Handle the escapes of quote in the filters

filter-parser/src/error.rs

@@ -82,7 +82,7 @@ impl<'a> Error<'a> {
     pub fn char(self) -> char {
         match self.kind {
             ErrorKind::Char(c) => c,
-            _ => panic!("Internal filter parser error"),
+            error => panic!("Internal filter parser error: {:?}", error),
         }
     }
 }

filter-parser/src/lib.rs

@@ -62,43 +62,53 @@ pub type Span<'a> = LocatedSpan<&'a str, &'a str>;
 type IResult<'a, Ret> = nom::IResult<Span<'a>, Ret, Error<'a>>;
 
 #[derive(Debug, Clone, Eq)]
-pub struct Token<'a>(Span<'a>);
+pub struct Token<'a> {
+    /// The token in the original input, it should be used when possible.
+    span: Span<'a>,
+    /// If you need to modify the original input you can use the `value` field
+    /// to store your modified input.
+    value: Option<String>,
+}
 
 impl<'a> Deref for Token<'a> {
     type Target = &'a str;
 
     fn deref(&self) -> &Self::Target {
-        &self.0
+        &self.span
     }
 }
 
 impl<'a> PartialEq for Token<'a> {
     fn eq(&self, other: &Self) -> bool {
-        self.0.fragment() == other.0.fragment()
+        self.span.fragment() == other.span.fragment()
     }
 }
 
 impl<'a> Token<'a> {
-    pub fn new(position: Span<'a>) -> Self {
-        Self(position)
+    pub fn new(span: Span<'a>, value: Option<String>) -> Self {
+        Self { span, value }
+    }
+
+    pub fn value(&self) -> &str {
+        self.value.as_ref().map_or(&self.span, |value| value)
     }
 
     pub fn as_external_error(&self, error: impl std::error::Error) -> Error<'a> {
-        Error::new_from_external(self.0, error)
+        Error::new_from_external(self.span, error)
     }
 
     pub fn parse<T>(&self) -> Result<T, Error>
     where
         T: FromStr,
         T::Err: std::error::Error,
     {
-        self.0.parse().map_err(|e| self.as_external_error(e))
+        self.span.parse().map_err(|e| self.as_external_error(e))
     }
 }
 
 impl<'a> From<Span<'a>> for Token<'a> {
     fn from(span: Span<'a>) -> Self {
-        Self(span)
+        Self { span, value: None }
     }
 }
 

filter-parser/src/value.rs

@@ -1,14 +1,21 @@
 use nom::branch::alt;
-use nom::bytes::complete::{take_till, take_while, take_while1};
-use nom::character::complete::{char, multispace0};
+use nom::bytes::complete::{escaped, take_till, take_while, take_while1};
+use nom::character::complete::{char, multispace0, none_of, one_of};
 use nom::combinator::cut;
 use nom::sequence::{delimited, terminated};
 
 use crate::error::NomErrorExt;
 use crate::{parse_geo_point, parse_geo_radius, Error, ErrorKind, IResult, Span, Token};
 
+/// This function goes through all chacaters in the [Span], if it finds any escaped character (`\`).
+/// It generate a new string with all `\` removed from the [Span].
+fn unescape(buf: Span, char_to_escape: char) -> Option<String> {
+    let to_escape = format!("\\{}", char_to_escape);
+    buf.contains(&to_escape).then(|| buf.replace(&to_escape, &char_to_escape.to_string()))
+}
+
 /// value          = WS* ~ ( word | singleQuoted | doubleQuoted) ~ WS*
-pub fn parse_value(input: Span) -> IResult<Token> {
+pub fn parse_value<'a>(input: Span<'a>) -> IResult<Token<'a>> {
     // to get better diagnostic message we are going to strip the left whitespaces from the input right now
     let (input, _) = take_while(char::is_whitespace)(input)?;
 
@@ -30,12 +37,20 @@ pub fn parse_value(input: Span) -> IResult<Token> {
         _ => (),
     }
 
-    // singleQuoted   = "'" .* all but quotes "'"
-    let simple_quoted = take_till(|c: char| c == '\'');
-    // doubleQuoted   = "\"" (word | spaces)* "\""
-    let double_quoted = take_till(|c: char| c == '"');
+    // singleQuoted   = "'" .* all but non-escaped quotes "'"
+    let simple_quoted = |input: Span<'a>| -> IResult<Token<'a>> {
+        escaped(none_of(r#"\'"#), '\\', one_of(r#"\'"#))(input)
+            .map(|(s, t)| (s, Token::new(t, unescape(t, '\''))))
+    };
+    // doubleQuoted   = "\"" .* all but non-escaped double quotes "\""
+    let double_quoted = |input: Span<'a>| -> IResult<Token<'a>> {
+        escaped(none_of(r#"\""#), '\\', one_of(r#"\""#))(input)
+            .map(|(s, t)| (s, Token::new(t, unescape(t, '"'))))
+    };
     // word           = (alphanumeric | _ | - | .)+
-    let word = take_while1(is_value_component);
+    let word = |input: Span<'a>| -> IResult<Token<'a>> {
+        take_while1(is_value_component)(input).map(|(s, t)| (s, t.into()))
+    };
 
     // this parser is only used when an error is encountered and it parse the
     // largest string possible that do not contain any “language” syntax.
@@ -54,7 +69,7 @@ pub fn parse_value(input: Span) -> IResult<Token> {
         )),
         multispace0,
     )(input)
-    .map(|(s, t)| (s, t.into()))
+    // .map(|(s, t)| (s, t.into()))
     // if we found nothing in the alt it means the user specified something that was not recognized as a value
     .map_err(|e: nom::Err<Error>| {
         e.map_err(|_| Error::new_from_kind(error_word(input).unwrap().1, ErrorKind::ExpectedValue))
@@ -81,7 +96,7 @@ pub mod test {
     use crate::tests::rtok;
 
     #[test]
-    fn name() {
+    fn test_span() {
         let test_case = [
             ("channel", rtok("", "channel")),
             (".private", rtok("", ".private")),
@@ -102,6 +117,7 @@ pub mod test {
             ("\"cha'nnel\"", rtok("'", "cha'nnel")),
             ("\"cha'nnel\"", rtok("'", "cha'nnel")),
             ("I'm tamo", rtok("'m tamo", "I")),
+            ("\"I'm \\\"super\\\" tamo\"", rtok("\"", "I'm \\\"super\\\" tamo")),
         ];
 
         for (input, expected) in test_case {
@@ -114,8 +130,94 @@ pub mod test {
                 expected,
                 result.unwrap_err()
             );
-            let value = result.unwrap().1;
-            assert_eq!(value, expected, "Filter `{}` failed.", input);
+            let token = result.unwrap().1;
+            assert_eq!(token, expected, "Filter `{}` failed.", input);
+        }
+    }
+
+    #[test]
+    fn test_unescape() {
+        assert_eq!(unescape(Span::new_extra("aaa", ""), '"'), None);
+        assert_eq!(unescape(Span::new_extra(r#""aaa""#, ""), '"'), None);
+        assert_eq!(unescape(Span::new_extra(r#"a\aa"#, ""), '"'), None);
+        assert_eq!(unescape(Span::new_extra(r#"a\\aa"#, ""), '"'), None);
+        assert_eq!(unescape(Span::new_extra(r#"\\\\\\"#, ""), '"'), None);
+        // double quote
+        assert_eq!(
+            unescape(Span::new_extra(r#"Hello \"World\""#, ""), '"'),
+            Some(r#"Hello "World""#.to_string())
+        );
+        assert_eq!(
+            unescape(Span::new_extra(r#"Hello \\\"World\\\""#, ""), '"'),
+            Some(r#"Hello \\"World\\""#.to_string())
+        );
+        // simple quote
+        assert_eq!(
+            unescape(Span::new_extra(r#"Hello \'World\'"#, ""), '\''),
+            Some(r#"Hello 'World'"#.to_string())
+        );
+        assert_eq!(
+            unescape(Span::new_extra(r#"Hello \\\'World\\\'"#, ""), '\''),
+            Some(r#"Hello \\'World\\'"#.to_string())
+        );
+    }
+
+    #[test]
+    fn test_value() {
+        let test_case = [
+            // (input, expected value, if a string was generated to hold the new value)
+            ("channel", "channel", false),
+            // All the base test, no escaped string should be generated
+            (".private", ".private", false),
+            ("I-love-kebab", "I-love-kebab", false),
+            ("but_snakes_is_also_good", "but_snakes_is_also_good", false),
+            ("parens(", "parens", false),
+            ("parens)", "parens", false),
+            ("not!", "not", false),
+            ("    channel", "channel", false),
+            ("channel     ", "channel", false),
+            ("    channel     ", "channel", false),
+            ("'channel'", "channel", false),
+            ("\"channel\"", "channel", false),
+            ("'cha)nnel'", "cha)nnel", false),
+            ("'cha\"nnel'", "cha\"nnel", false),
+            ("\"cha'nnel\"", "cha'nnel", false),
+            ("\" some spaces \"", " some spaces ", false),
+            ("\"cha'nnel\"", "cha'nnel", false),
+            ("\"cha'nnel\"", "cha'nnel", false),
+            ("I'm tamo", "I", false),
+            // All the tests with escaped string
+            (r#""\\""#, r#"\\"#, false),
+            (r#""\\\\\\""#, r#"\\\\\\"#, false),
+            (r#""aa\\aa""#, r#"aa\\aa"#, false),
+            // with double quote
+            (r#""Hello \"world\"""#, r#"Hello "world""#, true),
+            (r#""Hello \\\"world\\\"""#, r#"Hello \\"world\\""#, true),
+            (r#""I'm \"super\" tamo""#, r#"I'm "super" tamo"#, true),
+            // with simple quote
+            (r#"'Hello \'world\''"#, r#"Hello 'world'"#, true),
+            (r#"'Hello \\\'world\\\''"#, r#"Hello \\'world\\'"#, true),
+            (r#"'I\'m "super" tamo'"#, r#"I'm "super" tamo"#, true),
+        ];
+
+        for (input, expected, escaped) in test_case {
+            let input = Span::new_extra(input, input);
+            let result = parse_value(input);
+
+            assert!(
+                result.is_ok(),
+                "Filter `{:?}` was supposed to be parsed but failed with the following error: `{}`",
+                expected,
+                result.unwrap_err()
+            );
+            let token = result.unwrap().1;
+            assert_eq!(
+                token.value.is_some(),
+                escaped,
+                "Filter `{}` was not supposed to be escaped",
+                input
+            );
+            assert_eq!(token.value(), expected, "Filter `{}` failed.", input);
         }
     }