From 7bd5fbcd65b8631d7a0da48b3105dffbfc1bc80c Mon Sep 17 00:00:00 2001 From: loicgasser Date: Fri, 4 Aug 2017 14:40:12 +0000 Subject: [PATCH 1/6] Add support for nested includes --- mappyfile/parser.py | 66 +++++++++---------- mappyfile/utils.py | 24 +++---- tests/samples/include1_nested_path.map | 4 ++ .../include/include3_nested_path.map | 1 + .../mapfile_include/include2_nested_path.map | 4 ++ tests/test_sample_maps.py | 17 ++++- 6 files changed, 69 insertions(+), 47 deletions(-) create mode 100644 tests/samples/include1_nested_path.map create mode 100644 tests/samples/mapfile_include/include/include3_nested_path.map create mode 100644 tests/samples/mapfile_include/include2_nested_path.map diff --git a/mappyfile/parser.py b/mappyfile/parser.py index 0678a9d..1471e5c 100644 --- a/mappyfile/parser.py +++ b/mappyfile/parser.py @@ -1,6 +1,6 @@ import os, logging from io import open -from lark import Lark, ParseError +from lark import Lark import re try: @@ -9,52 +9,54 @@ # Python3 from io import StringIO + class Parser(object): - def __init__(self, cwd="", expand_includes=True, add_linebreaks=True): - self.cwd = cwd + def __init__(self, expand_includes=True, add_linebreaks=True): self.expand_includes = expand_includes self.add_linebreaks = add_linebreaks self.g = self.load_grammar("mapfile.g") + self._nested_include = 0 - def load_grammar(self, grammar_file): - gf = os.path.join(os.path.dirname(__file__), grammar_file) grammar_text = open(gf).read() + return Lark(grammar_text, parser="earley", lexer="standard") - return Lark(grammar_text, parser='earley', lexer='standard') - - def strip_quotes(self, s): + def _strip_quotes(self, s): + s = s[:s.index('#')] if '#' in s else s return s.strip("'").strip('"') - def load_includes(self, text): - + def load_includes(self, text, fn=None): + # Per default use working directory of the process + if fn is None: + fn = os.getcwd() lines = text.split('\n') includes = {} - + include_discovered = False for idx, l in enumerate(lines): - if l.strip().lower().startswith('include'): - if '#' in l: - l = l[:l.index('#')] - - parts = [p for p in l.split()] - - assert (len(parts) == 2) - assert (parts[0].lower() == 'include') - fn = os.path.join(self.cwd, self.strip_quotes(parts[1])) + if l.strip().lower().startswith("include"): + if not include_discovered: + include_discovered = True + self._nested_include += 1 + if self._nested_include > 5: + raise Exception("Maximum nested include exceeded! (MaxNested=5)") + + inc, inc_file_path = l.split() + inc_file_path = self._strip_quotes(inc_file_path) + if not os.path.isabs(inc_file_path): + inc_file_path = os.path.join(os.path.dirname(fn), inc_file_path) try: - include_text = self.open_file(fn) + include_text = self.open_file(inc_file_path) except IOError as ex: - logging.warning("Include file '%s' not found", fn) + logging.warning("Include file '%s' not found", inc_file_path) raise ex # recursively load any further includes - includes[idx] = self.load_includes(include_text) - + includes[idx] = self.load_includes(include_text, fn=inc_file_path) + for idx, txt in includes.items(): lines.pop(idx) # remove the original include lines.insert(idx, txt) - return '\n'.join(lines) def open_file(self, fn): @@ -66,11 +68,9 @@ def open_file(self, fn): raise def parse_file(self, fn): - - self.cwd = os.path.dirname(fn) - + self._nested_include = 0 text = self.open_file(fn) - return self.parse(text) + return self.parse(text, fn=fn) def _add_linebreaks(self, text): """ @@ -87,12 +87,12 @@ def _add_linebreaks(self, text): return "\n".join(new_lines) - def parse(self, text): - + def parse(self, text, fn=None): + self._nested_include = 0 if self.expand_includes == True: - text = self.load_includes(text) + text = self.load_includes(text, fn=fn) if self.add_linebreaks: text = self._add_linebreaks(text) - return self.g.parse(text) \ No newline at end of file + return self.g.parse(text) diff --git a/mappyfile/utils.py b/mappyfile/utils.py index a4e8240..ddd339f 100644 --- a/mappyfile/utils.py +++ b/mappyfile/utils.py @@ -3,36 +3,36 @@ from mappyfile.pprint import PrettyPrinter import codecs -def load(fn, cwd=None): - p = Parser(cwd=cwd) +def load(fn, expand_includes=True, add_linebreaks=True): + p = Parser(expand_includes=expand_includes, add_linebreaks=add_linebreaks) ast = p.parse_file(fn) m = MapfileToDict() d = m.transform(ast) + return d - return d -def loads(s, cwd="", expand_includes=True): - p = Parser(cwd=cwd, expand_includes=expand_includes) +def loads(s, expand_includes=True, add_linebreaks=True): + p = Parser(expand_includes=expand_includes, add_linebreaks=add_linebreaks) ast = p.parse(s) m = MapfileToDict() d = m.transform(ast) + return d - return d def write(d, output_file, indent=4): - map_string = _pprint(d, indent) _save(output_file, map_string) - return output_file + def dumps(d): return _pprint(d) + def find(lst, key, value): """ - When looking for an item by value also check for the value + When looking for an item by value also check for the value surrounded by apostrophes """ obj = __find__(lst, key, value) @@ -47,19 +47,21 @@ def find(lst, key, value): return obj + def __find__(lst, key, value): return next((item for item in lst if item[key.lower()] == value), None) + def findall(lst, key, value): possible_values = ("'%s'" % value, '"%s"' % value) - return (item for item in lst if item[key.lower()] in possible_values) -def _save(output_file, map_string): +def _save(output_file, map_string): with codecs.open(output_file, "w", encoding="utf-8") as f: f.write(map_string) + def _pprint(d, indent=4): pp = PrettyPrinter(indent=indent) return pp.pprint(d) diff --git a/tests/samples/include1_nested_path.map b/tests/samples/include1_nested_path.map new file mode 100644 index 0000000..54e0aec --- /dev/null +++ b/tests/samples/include1_nested_path.map @@ -0,0 +1,4 @@ +MAP + NAME 'include_test' + INCLUDE 'mapfile_include/include2_nested_path.map' +END diff --git a/tests/samples/mapfile_include/include/include3_nested_path.map b/tests/samples/mapfile_include/include/include3_nested_path.map new file mode 100644 index 0000000..14f55ac --- /dev/null +++ b/tests/samples/mapfile_include/include/include3_nested_path.map @@ -0,0 +1 @@ +NAME 'test' diff --git a/tests/samples/mapfile_include/include2_nested_path.map b/tests/samples/mapfile_include/include2_nested_path.map new file mode 100644 index 0000000..7b63794 --- /dev/null +++ b/tests/samples/mapfile_include/include2_nested_path.map @@ -0,0 +1,4 @@ +LAYER + NAME 'include_test' + INCLUDE 'include/include3_nested_path.map' +END diff --git a/tests/test_sample_maps.py b/tests/test_sample_maps.py index 43fe36c..08cf129 100644 --- a/tests/test_sample_maps.py +++ b/tests/test_sample_maps.py @@ -21,18 +21,29 @@ def test_all_maps(): def test_includes(): p = Parser() - + ast = p.parse_file('./tests/samples/include1.map') m = MapfileToDict() d = (m.transform(ast)) # works print(mappyfile.dumps(d)) -def run_tests(): + +def test_includes_nested_path(): + p = Parser() + + ast = p.parse_file('./tests/samples/include1_nested_path.map') + m = MapfileToDict() + + d = (m.transform(ast)) # works + print(mappyfile.dumps(d)) + + +def run_tests(): pytest.main(["tests/test_sample_maps.py"]) if __name__ == '__main__': logging.basicConfig(level=logging.INFO) test_all_maps() #run_tests() - print("Done!") \ No newline at end of file + print("Done!") From ce9ebfb4ae7be095e903a07c99bc59065d5c7fb3 Mon Sep 17 00:00:00 2001 From: Seth G Date: Tue, 15 Aug 2017 17:59:17 +0200 Subject: [PATCH 2/6] Update parser.py --- mappyfile/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mappyfile/parser.py b/mappyfile/parser.py index 95f910b..62670a5 100644 --- a/mappyfile/parser.py +++ b/mappyfile/parser.py @@ -94,7 +94,7 @@ def _add_linebreaks(self, text): def parse(self, text, fn=None): self._nested_include = 0 - if self.expand_includes == True: + if self.expand_includes: text = self.load_includes(text, fn=fn) if self.add_linebreaks: From 70a3b66d84bf59285e06a2586ff42aec06dc4dda Mon Sep 17 00:00:00 2001 From: Seth G Date: Tue, 15 Aug 2017 17:59:53 +0200 Subject: [PATCH 3/6] Update utils.py --- mappyfile/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mappyfile/utils.py b/mappyfile/utils.py index 8af3f77..ddd339f 100644 --- a/mappyfile/utils.py +++ b/mappyfile/utils.py @@ -3,6 +3,7 @@ from mappyfile.pprint import PrettyPrinter import codecs + def load(fn, expand_includes=True, add_linebreaks=True): p = Parser(expand_includes=expand_includes, add_linebreaks=add_linebreaks) ast = p.parse_file(fn) @@ -10,6 +11,7 @@ def load(fn, expand_includes=True, add_linebreaks=True): d = m.transform(ast) return d + def loads(s, expand_includes=True, add_linebreaks=True): p = Parser(expand_includes=expand_includes, add_linebreaks=add_linebreaks) ast = p.parse(s) @@ -17,6 +19,7 @@ def loads(s, expand_includes=True, add_linebreaks=True): d = m.transform(ast) return d + def write(d, output_file, indent=4): map_string = _pprint(d, indent) _save(output_file, map_string) From 0799b468e5948349247153b36b94b2cdfeb425d3 Mon Sep 17 00:00:00 2001 From: Seth G Date: Tue, 15 Aug 2017 18:07:57 +0200 Subject: [PATCH 4/6] Update test_sample_maps.py --- tests/test_sample_maps.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_sample_maps.py b/tests/test_sample_maps.py index 1eba944..e18c092 100644 --- a/tests/test_sample_maps.py +++ b/tests/test_sample_maps.py @@ -27,7 +27,7 @@ def test_includes(): ast = p.parse_file('./tests/samples/include1.map') m = MapfileToDict() - d = (m.transform(ast)) # works + d = (m.transform(ast)) # works print(mappyfile.dumps(d)) @@ -48,4 +48,5 @@ def run_tests(): if __name__ == '__main__': logging.basicConfig(level=logging.INFO) test_all_maps() - print("Done!") \ No newline at end of file + print("Done!") + From d9aea6955e6bb47fac6bcc8639cc6ee38f76b251 Mon Sep 17 00:00:00 2001 From: Seth G Date: Tue, 15 Aug 2017 22:57:22 +0200 Subject: [PATCH 5/6] Update test_sample_maps.py --- tests/test_sample_maps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sample_maps.py b/tests/test_sample_maps.py index e18c092..d2f988b 100644 --- a/tests/test_sample_maps.py +++ b/tests/test_sample_maps.py @@ -37,7 +37,7 @@ def test_includes_nested_path(): ast = p.parse_file('./tests/samples/include1_nested_path.map') m = MapfileToDict() - d = (m.transform(ast)) # works + d = (m.transform(ast)) # works print(mappyfile.dumps(d)) From d2f487c2b6253efd0059bb07927699edf4219b9b Mon Sep 17 00:00:00 2001 From: Seth G Date: Tue, 15 Aug 2017 23:07:13 +0200 Subject: [PATCH 6/6] Update test_sample_maps.py