Support PEP 498, Literal String Interpolation

PyCQA · Aug 18, 2017 · eeabbc2 · eeabbc2
1 parent 08c7561
commit eeabbc2
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 3 deletions.
diff --git a/baron/formatting_grouper.py b/baron/formatting_grouper.py
@@ -65,6 +65,8 @@ class UnExpectedSpaceToken(BaronError):
 STRING = (
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "UNICODE_STRING",
     "UNICODE_RAW_STRING",
     "BINARY_STRING",

diff --git a/baron/grammator.py b/baron/grammator.py
@@ -655,10 +655,12 @@ def strings_string(pack):
     # TODO tests those other kind of strings
     @pg.production("string : STRING")
     @pg.production("string : RAW_STRING")
+    @pg.production("string : INTERPOLATED_STRING")
     @pg.production("string : UNICODE_STRING")
     @pg.production("string : BINARY_STRING")
     @pg.production("string : UNICODE_RAW_STRING")
     @pg.production("string : BINARY_RAW_STRING")
+    @pg.production("string : INTERPOLATED_RAW_STRING")
     def string(pack):
         (string_,) = pack
         return [{

diff --git a/baron/grouper.py b/baron/grouper.py
@@ -51,9 +51,9 @@ def group_generator(sequence):
             current += next(iterator)
         if current in to_group_keys and matching_found(to_group, current, iterator.show_next()):
             current += next(iterator)
-        if current in list('uUrRbB') and str(iterator.show_next()).startswith(('"', "'")):
+        if current in list('uUfFrRbB') and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
-        if str(current).lower() in ["ur", "br"] and str(iterator.show_next()).startswith(('"', "'")):
+        if str(current).lower() in ["ur", "br", "fr"] and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
         if any([re.match(x, current) for x in (r'^\d+[eE]$', r'^\d+\.\d*[eE]$', r'^\.\d+[eE]$')]):
             current += next(iterator)

diff --git a/baron/inner_formatting_grouper.py b/baron/inner_formatting_grouper.py
@@ -35,6 +35,8 @@ class GroupingError(BaronError):
     # TODO test everything bellow
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "BINARY_STRING",
     "BINARY_RAW_STRING",
     "UNICODE_STRING",

diff --git a/baron/tokenizer.py b/baron/tokenizer.py
@@ -78,10 +78,12 @@ class UnknowItem(BaronError):
     (r'(\s|\\\n|\\\r\n)+', 'SPACE'),
     (r'["\'](.|\n|\r)*["\']', 'STRING'),
     (r'[uU]["\'](.|\n|\r)*["\']', 'UNICODE_STRING'),
+    (r'[fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_STRING'),
     (r'[rR]["\'](.|\n|\r)*["\']', 'RAW_STRING'),
     (r'[bB]["\'](.|\n|\r)*["\']', 'BINARY_STRING'),
     (r'[uU][rR]["\'](.|\n|\r)*["\']', 'UNICODE_RAW_STRING'),
     (r'[bB][rR]["\'](.|\n|\r)*["\']', 'BINARY_RAW_STRING'),
+    (r'[fF][rR]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
 )
 
 

diff --git a/tests/test_formatting_grouper.py b/tests/test_formatting_grouper.py
@@ -2889,7 +2889,7 @@ def test_strings():
     """
     I don't this because python allow to write stuff like 'qsd' rb"qsd" u'pouet'
     """
-    for i in ('STRING', 'RAW_STRING', 'UNICODE_STRING', 'UNICODE_RAW_STRING', 'BINARY_STRING', 'BINARY_RAW_STRING'):
+    for i in ('STRING', 'RAW_STRING', 'UNICODE_STRING', 'INTERPOLATED_STRING', 'UNICODE_RAW_STRING', 'BINARY_STRING', 'INTERPOLATED_RAW_STRING', 'BINARY_RAW_STRING'):
         group([
             ('SPACE', ' '),
             (i, 'dummy'),

diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -289,6 +289,27 @@ def test_unicode_string():
     match("U'''pouet pouet'''", "UNICODE_STRING")
 
 
+def test_interpolated_string():
+    match("f'He said his name is {name!r}.'", 'INTERPOLATED_STRING')
+    match("f'The value is {value}.'", 'INTERPOLATED_STRING')
+    match('F"He said his name is {name!r}."', 'INTERPOLATED_STRING')
+    match('f"The value is {value}."', 'INTERPOLATED_STRING')
+    match("F'{date} was on a {date:%A}'", 'INTERPOLATED_STRING')
+    match("f'a={d[\"a\"]}'", 'INTERPOLATED_STRING')
+    match("F'{(lambda x: x*2)(3)}'", 'INTERPOLATED_STRING')
+
+
+def test_interpolated_raw_string():
+    match("fr'He said his name is {name!r}.'", 'INTERPOLATED_RAW_STRING')
+    match("fr'The value is {value}.'", 'INTERPOLATED_RAW_STRING')
+    match('Fr"He said his name is {name!r}."', 'INTERPOLATED_RAW_STRING')
+    match('fR"The value is {value}."', 'INTERPOLATED_RAW_STRING')
+    match("FR'{date} was on a {date:%A}'", 'INTERPOLATED_RAW_STRING')
+    match("fr'a={d[\"a\"]}'", 'INTERPOLATED_RAW_STRING')
+    match("FR'{(lambda x: x*2)(3)}'", 'INTERPOLATED_RAW_STRING')
+    match("FR'{(lambda x: x*2)(3)}'", 'INTERPOLATED_RAW_STRING')
+
+
 def test_raw_string():
     match('r"pouet pouet"', 'RAW_STRING')
     match("r'pouet pouet'", "RAW_STRING")