diff --git a/README.md b/README.md index 94d3ac8..1e8ab5d 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ Options: contains more than one ApiGateway resource) -f, --format [yaml|json] output format [default: yaml] -i, --indent INTEGER if output format is json, specify indentation + -d, --debug if enabled, show debug logs + --no-ignore if set, deforest will export paths marked as + ignored --version Show the version and exit. --help Show this message and exit. ``` @@ -33,6 +36,97 @@ Options: Version 0.1.1 and later supports CloudFormation templates as input. If more than one API Gateway is part of the template, the `--outfile` flag will be ignored. +## Hide paths + +Version 0.2.0 introduced support for deforest to ignore certain paths. If you specify `x-deforest-ignore: true` anywhere in your specification, deforest will not extract its _parent_ node to the end results. Example: + +```yaml +paths: + "/validation": + post: + responses: + "200": + schema: + type: array + items: + "$ref": "#/definitions/Error" + headers: + test-method-response-header: + type: string + get: + x-deforest-ignore: true + parameters: + - name: q1 + in: query + required: true + responses: + "200": + schema: + type: array + items: + "$ref": "#/definitions/Error" + headers: + test-method-response-header: + type: string +``` + +will result in + +```yaml +paths: + /validation: + post: + responses: + "200": + headers: + test-method-response-header: + type: string + schema: + items: + $ref: "#/definitions/Error" + type: array +``` + +If we'd written this: + +```yaml +paths: + "/validation": + x-deforest-ignore: true + post: + responses: + "200": + schema: + type: array + items: + "$ref": "#/definitions/Error" + headers: + test-method-response-header: + type: string + get: + parameters: + - name: q1 + in: query + required: true + responses: + "200": + schema: + type: array + items: + "$ref": "#/definitions/Error" + headers: + test-method-response-header: + type: string +``` + +we'd get an empty result since the _parent_ node is removed: + +```yaml +paths: {} +``` + +If `x-deforest-ignore: false`, or missing, the node will be extracted as usual. If the runtime flag `--no-ignore` is set, the nodes will be extracted as usual as well. + # Limitations The output file looses its order of the keys in the file, which shouldn't affect you if you're using a converter to create a graphical documentation/specification - but can be confusing if you have a specific internal order you wish to keep. diff --git a/deforest/__init__.py b/deforest/__init__.py index e69de29..57c2dd0 100644 --- a/deforest/__init__.py +++ b/deforest/__init__.py @@ -0,0 +1,2 @@ +VERSION = "0.1.1" +LOGGER = "deforest" diff --git a/deforest/cfcleaner.py b/deforest/cfcleaner.py deleted file mode 100644 index f40d0b2..0000000 --- a/deforest/cfcleaner.py +++ /dev/null @@ -1,97 +0,0 @@ -from tags import GetAttTag, SubTag, RefTag -import yaml -import re - -class CFCleaner(): - keys = ["x-amazon"] - filedata = None - raw = {} - processed = None - KEY_TYPE = "Type" - KEY_RESOURCES = "Resources" - KEY_PROPERTIES = "Properties" - KEY_BODY = "Body" - TYPE_APIGW = "AWS::ApiGateway::RestApi" - - multi_raw = [] - multi_processed = [] - - def __init__(self,data): - self.filedata = data - - def convert(self): - self._enable_custom_tags() - self._load() - self._clean_all_keys() - self._dump_all() - return self.multi_processed - - def number_results(self): - return len(self.multi_processed) - - def _clean_all_keys(self): - self._identify_apigw(self.raw) - - def _identify_apigw(self,v): - resources = v[CFCleaner.KEY_RESOURCES] - for k in resources: - if isinstance(resources[k], dict): - if self._is_apigw(resources[k]) and self._has_properties_and_body(resources[k]): - self.multi_raw.append(resources[k][CFCleaner.KEY_PROPERTIES][CFCleaner.KEY_BODY]) - - for raw in self.multi_raw: - self._cleanup_keys(raw) - - def _cleanup_keys(self,v): - for k in v.keys(): - if any(m in k for m in self.keys): - del v[k] - else: - if isinstance(v[k], dict): - self._cleanup_keys(v[k]) - - def _is_apigw(self,node): - if CFCleaner.KEY_TYPE in node: - if node[CFCleaner.KEY_TYPE] == CFCleaner.TYPE_APIGW: - return True - return False - - def _has_properties_and_body(self,node): - return CFCleaner.KEY_PROPERTIES in node and CFCleaner.KEY_BODY in node[CFCleaner.KEY_PROPERTIES] - - def _namify(self, title, version): - s = "{}-{}".format(title.lower(),version.lower()) - s = re.sub(r"\s+", '-', s) - return s - - def get_title_and_version_all(self,index): - title = self.multi_raw[index]["info"]["title"] or "no-title" - version = self.multi_raw[index]["info"]["version"] or "no-version" - return self._namify(title,version) - - def get_title_and_version(self): - return self.get_title_and_version(0) - - def _enable_custom_tags(self): - yaml.SafeLoader.add_constructor('!GetAtt', GetAttTag.from_yaml) - yaml.SafeLoader.add_constructor('!Sub', SubTag.from_yaml) - yaml.SafeLoader.add_constructor('!Ref', RefTag.from_yaml) - yaml.SafeDumper.add_multi_representer(GetAttTag, GetAttTag.to_yaml) - yaml.SafeDumper.add_multi_representer(SubTag, SubTag.to_yaml) - yaml.SafeDumper.add_multi_representer(RefTag, RefTag.to_yaml) - - def _load(self): - self.raw = yaml.safe_load(self.filedata) - - def _dump(self): - self.processed = yaml.safe_dump(self.multi_raw[0]) - - def _dump_all(self): - for r in self.multi_raw: - self.multi_processed.append(yaml.safe_dump(r)) - - def get_raw(self,index): - return self.multi_raw[0] - - def get_raw_all(self,index): - return self.multi_raw[index] \ No newline at end of file diff --git a/deforest/cleaner.py b/deforest/cleaner.py deleted file mode 100644 index 92bb743..0000000 --- a/deforest/cleaner.py +++ /dev/null @@ -1,66 +0,0 @@ -from tags import GetAttTag, SubTag, RefTag -import yaml -import re - -class DeforestCleaner(): - keys = ["x-amazon"] - filedata = None - raw = {} - processed = None - - def __init__(self, data): - self.filedata = data - - def _namify(self, title, version): - s = "{}-{}".format(title.lower(),version.lower()) - s = re.sub(r"\s+", '-', s) - return s - - def get_title_and_version(self): - title = self.raw["info"]["title"] or "no-title" - version = self.raw["info"]["version"] or "no-version" - return self._namify(title,version) - - def get_title_and_version_all(self,index): - return self.get_title_and_version() - - def get_raw(self): - return self.raw - - def get_raw_all(self,index): - return self.raw - - def convert(self): - self._enable_custom_tags() - self._load() - self._clean_all_keys() - self._dump() - return [self.processed] - - def _clean_all_keys(self): - self._cleanup_keys(self.raw) - - def _cleanup_keys(self, v): - for k in v.keys(): - if any(m in k for m in self.keys): - del v[k] - else: - if isinstance(v[k], dict): - self._cleanup_keys(v[k]) - - def _enable_custom_tags(self): - yaml.SafeLoader.add_constructor('!GetAtt', GetAttTag.from_yaml) - yaml.SafeLoader.add_constructor('!Sub', SubTag.from_yaml) - yaml.SafeLoader.add_constructor('!Ref', RefTag.from_yaml) - yaml.SafeDumper.add_multi_representer(GetAttTag, GetAttTag.to_yaml) - yaml.SafeDumper.add_multi_representer(SubTag, SubTag.to_yaml) - yaml.SafeDumper.add_multi_representer(RefTag, RefTag.to_yaml) - - def _load(self): - self.raw = yaml.safe_load(self.filedata) - - def _dump(self): - self.processed = yaml.safe_dump(self.raw) - - def number_results(self): - return 1 \ No newline at end of file diff --git a/deforest/cleaners.py b/deforest/cleaners.py new file mode 100644 index 0000000..2ad618c --- /dev/null +++ b/deforest/cleaners.py @@ -0,0 +1,90 @@ +import logging + + +class CloudFormationCleaner: + KEY_CF_FIELD = "AWSTemplateFormatVersion" + KEY_CF_RESOURCES = "Resources" + KEY_CF_BODY = "Body" + KEY_CF_TYPE = "Type" + KEY_CF_PROPERTIES = "Properties" + CF_APIGW_TYPE = "AWS::ApiGateway::RestApi" + + def __init__(self, caller): + logging.debug("created {}".format(self)) + self.caller = caller + + def clean(self): + logging.debug("{} clean".format(self)) + logging.debug("checking {}".format(self.caller.result)) + if not CloudFormationCleaner.KEY_CF_FIELD in self.caller.result: + self.caller.result = [self.caller.result] + return + rval = [] + copy = self.caller.result + res = copy[CloudFormationCleaner.KEY_CF_RESOURCES] + for k in res: + if isinstance(res[k], dict): + if self._is_apigw(res[k]) and self._has_body(res[k]): + logging.info( + "identified resource '{}' as an API Gateway".format(k)) + rval.append( + res[k][CloudFormationCleaner.KEY_CF_PROPERTIES][CloudFormationCleaner.KEY_CF_BODY]) + self.caller.result = rval + + def _is_apigw(self, node): + if CloudFormationCleaner.KEY_CF_TYPE in node: + return node[CloudFormationCleaner.KEY_CF_TYPE] == CloudFormationCleaner.CF_APIGW_TYPE + return False + + def _has_body(self, node): + return CloudFormationCleaner.KEY_CF_PROPERTIES in node and CloudFormationCleaner.KEY_CF_BODY in node[CloudFormationCleaner.KEY_CF_PROPERTIES] + + +class DefaultCleaner: + keys = ["x-amazon"] + copy = None + + def __init__(self, caller): + logging.debug("created {}".format(self)) + self.caller = caller + + def clean(self): + logging.debug("{} clean".format(self)) + logging.debug("checking {}".format(self.caller.result)) + self.copy = self.caller.result + for i in self.copy: + self._clean(i) + self.caller.result = self.copy + + def _clean(self, v): + for k in v.keys(): + if any(m in k for m in self.keys): + del v[k] + else: + if isinstance(v[k], dict): + self._clean(v[k]) + + +class IgnoreCleaner: + DEFOREST_IGNORE_KEY = "x-deforest-ignore" + copy = None + + def __init__(self, caller): + logging.debug("created {}".format(self)) + self.caller = caller + + def clean(self): + self.copy = self.caller.result + for i in self.copy: + self._clean(i) + self.caller.result = self.copy + + def _clean(self, v): + for k in v.keys(): + if isinstance(v[k], dict): + if IgnoreCleaner.DEFOREST_IGNORE_KEY in v[k]: + if v[k][IgnoreCleaner.DEFOREST_IGNORE_KEY] is True: + logging.debug("removing {}".format(v[k])) + del v[k] + else: + self._clean(v[k]) diff --git a/deforest/constant.py b/deforest/constant.py index 57c2dd0..99e4c7b 100644 --- a/deforest/constant.py +++ b/deforest/constant.py @@ -1,2 +1,4 @@ -VERSION = "0.1.1" +VERSION = "0.2.0" LOGGER = "deforest" +EXIT_NOTFOUND = 1 +EXIT_PARSEERR = 2 diff --git a/deforest/deforest.py b/deforest/deforest.py index bdb02fe..c6f9503 100644 --- a/deforest/deforest.py +++ b/deforest/deforest.py @@ -1,36 +1,52 @@ +import coloredlogs +import logging import click -import json +import sys +from constant import VERSION, LOGGER, EXIT_NOTFOUND +from filecleaner import ForestCleaner +from filecreator import FileCreator -from constant import VERSION, LOGGER -from cleaner import DeforestCleaner -from solution import Solution @click.command() @click.argument("infile") -@click.option("--outfile","-o", help="specify output file, default is ./-<version>.<format>, ignored if input is a CloudFormation template and the template contains more than one ApiGateway resource)") -@click.option("--format","-f", default="yaml", show_default=True, type=click.Choice(["yaml","json"]), help="output format") -@click.option("--indent", "-i", default=4,type=int, help="if output format is json, specify indentation") +@click.option("--outfile", "-o", help="specify output file, default is ./<title>-<version>.<format>, ignored if input is a CloudFormation template and the template contains more than one ApiGateway resource)") +@click.option("--format", "-f", default="yaml", show_default=True, type=click.Choice(["yaml", "json"]), help="output format") +@click.option("--indent", "-i", default=4, type=int, help="if output format is json, specify indentation") +@click.option("--debug", "-d", default=False, is_flag=True, help="if enabled, show debug logs") +@click.option("--no-ignore", default=False, is_flag=True, help="if set, deforest will export paths marked as ignored") @click.version_option(VERSION) -def main(infile,outfile,format,indent): - with open(infile, "r") as fh: - d = fh.read() - - cleaner = Solution(d).cleaner() - result = cleaner.convert() - - filename = None - if outfile and len(result) < 2: - filename = outfile - else: - print("output will be in {} files, ignoring --outfile flag setting".format(len(result))) - - for i,r in enumerate(result): - tfile = filename - if filename is None: - tfile = "{}.{}".format(cleaner.get_title_and_version_all(i),"json" if format == "json" else "yaml") - - with open(tfile,"w+") as fh: - if format == "json": - fh.write(json.dumps(cleaner.get_raw_all(i), indent=indent)) - else: - fh.write(r) +def main(infile, outfile, format, indent, debug, no_ignore): + set_log_level(debug) + logging.debug("parsing file '{}'".format(infile)) + d = read_file(infile) + logging.debug("read {} bytes from file".format(len(d))) + + f = ForestCleaner(d) + f.allow_ignored = no_ignore + cleaned = f.clean() + + logging.debug("expected output {} files".format(len(cleaned))) + + fw = FileCreator(cleaned) + fw.format = format + fw.filename = outfile + fw.write_to_file() + + +def set_log_level(debug): + log_name = logging.getLogger(LOGGER) + log_level = "INFO" + log_format = '%(levelname)s: %(message)s' + if debug: + log_level = "DEBUG" + coloredlogs.install(level=log_level, fmt=log_format) + + +def read_file(filename): + try: + with open(filename, "r") as fh: + return fh.read() + except IOError as e: + logging.error("could not read file '{}': {}".format( + filename, e.strerror)) + sys.exit(EXIT_NOTFOUND) diff --git a/deforest/filecleaner.py b/deforest/filecleaner.py new file mode 100644 index 0000000..8f7f193 --- /dev/null +++ b/deforest/filecleaner.py @@ -0,0 +1,74 @@ +import logging +from Queue import Queue +from tags import GetAttTag, SubTag, RefTag +import yaml +from cleaners import DefaultCleaner, CloudFormationCleaner, IgnoreCleaner +import sys +from constant import EXIT_PARSEERR + + +class ForestCleaner: + cleaner_queue = None + _result = None + _allow_ignored = False + + def __init__(self, data): + logging.debug("received {} bytes of data to clean".format(len(data))) + self.data = data + self._parse_data() + + def _parse_data(self): + self._enable_cf_tags() + logging.debug("parsing yaml") + try: + self.result = yaml.safe_load(self.data) + except yaml.scanner.ScannerError as e: + logging.error("could not parse file: {}".format(e)) + sys.exit(EXIT_PARSEERR) + + def _create_queue(self): + logging.debug("creating cleaner queue") + cleaner_queue = Queue() + logging.debug("created queue with size {}".format( + cleaner_queue.qsize())) + + cleaner_queue.put(CloudFormationCleaner(self)) + + cleaner_queue.put(DefaultCleaner(self)) + + if not self.allow_ignored: + cleaner_queue.put(IgnoreCleaner(self)) + else: + logging.info("allowing x-deforest-ignore paths") + self.cleaner_queue = cleaner_queue + + def clean(self): + self._create_queue() + while not self.cleaner_queue.empty(): + self.cleaner_queue.get().clean() + return self.result + + def _enable_cf_tags(self): + logging.debug("enabling CF tags") + yaml.SafeLoader.add_constructor('!GetAtt', GetAttTag.from_yaml) + yaml.SafeLoader.add_constructor('!Sub', SubTag.from_yaml) + yaml.SafeLoader.add_constructor('!Ref', RefTag.from_yaml) + yaml.SafeDumper.add_multi_representer(GetAttTag, GetAttTag.to_yaml) + yaml.SafeDumper.add_multi_representer(SubTag, SubTag.to_yaml) + yaml.SafeDumper.add_multi_representer(RefTag, RefTag.to_yaml) + + @property + def result(self): + return self._result + + @result.setter + def result(self, value): + self._result = value + + @property + def allow_ignored(self): + return self._allow_ignored + + @allow_ignored.setter + def allow_ignored(self, value): + self._allow_ignored = value diff --git a/deforest/filecreator.py b/deforest/filecreator.py new file mode 100644 index 0000000..99442cc --- /dev/null +++ b/deforest/filecreator.py @@ -0,0 +1,67 @@ +import logging +import re +import json +import yaml + + +class FileCreator: + _format = "yaml" + _indent = 4 + _filename = None + + def __init__(self, data): + logging.debug("creating {}".format(self)) + self.data = data + + def write_to_file(self): + logging.debug("writing {} results to file, format setting: {}".format( + len(self.data), self.format)) + + if len(self.data) > 1 and self.filename is not None: + logging.warning( + "using default deforest filenaming conventions since the output constists of more than one file") + self.filename = None + + for d in self.data: + fname = self.filename + if self.filename is None: + logging.debug("setting filename {}".format(self._filename(d))) + fname = self._filename(d) + with open(fname, "w+") as fh: + if self.format == "json": + fh.write(json.dumps(d, indent=self.indent)) + else: + yd = yaml.safe_dump(d) + fh.write(yd) + logging.info("saved to file {}".format(fname)) + + def _filename(self, content): + title = content["info"]["title"] or "no-title" + version = content["info"]["version"] or "no-version" + s = "{}-{}.{}".format(title.lower(), version.lower(), self.format) + s = re.sub(r"\s+", '-', s) + return s + + @property + def format(self): + return self._format + + @format.setter + def format(self, value): + self._format = value + + @property + def indent(self): + return self._indent + + @indent.setter + def indent(self, value): + self._indent = value + + @property + def filename(self): + return self._filename + + @filename.setter + def filename(self, value): + self._filename = value diff --git a/deforest/solution.py b/deforest/solution.py deleted file mode 100644 index 08e045b..0000000 --- a/deforest/solution.py +++ /dev/null @@ -1,41 +0,0 @@ -from tags import GetAttTag, SubTag, RefTag -import yaml -import re -from cleaner import DeforestCleaner -from cfcleaner import CFCleaner - -class Solution(): - raw = {} - filedata = None - processed = None - - def __init__(self, data): - self.filedata = data - - def _enable_custom_tags(self): - yaml.SafeLoader.add_constructor('!GetAtt', GetAttTag.from_yaml) - yaml.SafeLoader.add_constructor('!Sub', SubTag.from_yaml) - yaml.SafeLoader.add_constructor('!Ref', RefTag.from_yaml) - yaml.SafeDumper.add_multi_representer(GetAttTag, GetAttTag.to_yaml) - yaml.SafeDumper.add_multi_representer(SubTag, SubTag.to_yaml) - yaml.SafeDumper.add_multi_representer(RefTag, RefTag.to_yaml) - - def _load(self): - self.raw = yaml.safe_load(self.filedata) - - def _dump(self): - self.processed = yaml.safe_dump(self.raw) - - def cleaner(self): - self._enable_custom_tags() - self._load() - cleaner = self._identify_cleaner() - if cleaner is "default": - return DeforestCleaner(self.filedata) - if cleaner is "cloudformation": - return CFCleaner(self.filedata) - - def _identify_cleaner(self): - if "AWSTemplateFormatVersion" in self.raw: - return "cloudformation" - return "default" \ No newline at end of file diff --git a/deforest/tags.py b/deforest/tags.py index b5b5100..ac8acde 100644 --- a/deforest/tags.py +++ b/deforest/tags.py @@ -1,5 +1,6 @@ import yaml + class GetAttTag(yaml.YAMLObject): tag = u'!GetAtt' @@ -17,6 +18,7 @@ def from_yaml(cls, loader, node): def to_yaml(cls, dumper, data): return '' + class SubTag(yaml.YAMLObject): tag = u'!Sub' @@ -34,6 +36,7 @@ def from_yaml(cls, loader, node): def to_yaml(cls, dumper, data): return '' + class RefTag(yaml.YAMLObject): tag = u'!Ref' diff --git a/setup.py b/setup.py index c8a2154..dc6bca6 100644 --- a/setup.py +++ b/setup.py @@ -3,14 +3,14 @@ import os from deforest.constant import VERSION -with open("README.md","r") as fh: +with open("README.md", "r") as fh: long_desc = fh.read() setuptools.setup( name="deforest", version=VERSION, author="hawry", - entry_points = { + entry_points={ "console_scripts": ["deforest=deforest.deforest:main"] }, author_email="hawry@hawry.net", @@ -21,7 +21,8 @@ packages=setuptools.find_packages(), install_requires=[ "pyyaml==5.1.1", - "click==6.7" + "click==6.7", + "coloredlogs==10.0" ], classifiers=[ "Programming Language :: Python :: 2.7",