update deps and initial abnf support

Author: anweiss

Cargo.toml

@@ -25,7 +25,7 @@ chrono = { version = "0.4", optional = true }
 clap = { version = "2.33", optional = true }
 codespan-reporting = "0.11"
 hexf-parse = "0.1"
-itertools = "0.9"
+itertools = "0.10"
 lexical-core = "0.7"
 regex = { version = "1.4", default-features = false, features = ["std"] }
 regex-syntax = { version = "0.6", optional = true }
@@ -34,13 +34,16 @@ serde_cbor = { version = "0.11.1", optional = true, default-features = false, fe
 serde_json = { version = "1.0", optional = true, default-features = false }
 uriparse = { version = "0.6", optional = true }
 base64-url = { version = "1.4", optional = true }
+abnf_to_pest = "0.5"
+pest_meta = "2.1"
+pest_vm = "2.1"
 
 [dev-dependencies]
 indoc = "1.0"
 pretty_assertions = { git = "https://github.com/colin-kiegel/rust-pretty-assertions" }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-crossterm = { version = "0.18", optional = true }
+crossterm = { version = "0.19", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 console_error_panic_hook = "0.1"

src/token.rs

@@ -123,6 +123,14 @@ pub enum Token<'a> {
   /// Proposed control extension for numeric addition. See
   /// https://tools.ietf.org/html/draft-ietf-cbor-cddl-control-01#section-2.2.
   PLUS,
+  /// .abnf control operator
+  /// Proposed control extension for embedded ABNF as UTF-8. See
+  /// https://tools.ietf.org/html/draft-ietf-cbor-cddl-control-01#section-3
+  ABNF,
+  /// .abnfb control operator
+  /// Proposed control extension for embedded ABNF as a sequence of bytes. See
+  /// https://tools.ietf.org/html/draft-ietf-cbor-cddl-control-01#section-3
+  ABNFB,
 
   /// group to choice enumeration '&'
   GTOCHOICE,
@@ -488,6 +496,8 @@ impl<'a> fmt::Display for Token<'a> {
       Token::WITHIN => write!(f, ".within"),
       Token::CAT => write!(f, ".cat"),
       Token::PLUS => write!(f, ".plus"),
+      Token::ABNF => write!(f, ".abnf"),
+      Token::ABNFB => write!(f, ".abnfb"),
       Token::AND => write!(f, ".and"),
       Token::LT => write!(f, ".lt"),
       Token::LE => write!(f, ".le"),
@@ -568,6 +578,8 @@ pub fn lookup_control_from_str<'a>(ident: &str) -> Option<Token<'a>> {
     ".pcre" => Some(Token::PCRE),
     ".cat" => Some(Token::CAT),
     ".plus" => Some(Token::PLUS),
+    ".abnf" => Some(Token::ABNF),
+    ".abnfb" => Some(Token::ABNFB),
     _ => None,
   }
 }
@@ -605,6 +617,8 @@ pub fn control_str_from_token(t: &Token) -> Option<&'static str> {
     Token::PCRE => Some(".pcre"),
     Token::CAT => Some(".cat"),
     Token::PLUS => Some(".plus"),
+    Token::ABNF => Some(".abnf"),
+    Token::ABNFB => Some(".abnfb"),
     _ => None,
   }
 }

src/validator/control.rs

@@ -1,3 +1,5 @@
+use pest_meta::parser;
+
 use crate::{
   ast::{Identifier, Operator, RangeCtlOp, Rule, Type2, CDDL},
   token::ByteValue,
@@ -561,3 +563,68 @@ pub fn plus_operation<'a>(
 
   Ok(values)
 }
+
+pub fn validate_abnf(abnf: &str, target: &str) -> Result<(), String> {
+  if let Some(idx) = abnf.find('\n') {
+    let (rule, abnf) = abnf.split_at(idx);
+
+    let rules = abnf_to_pest::parse_abnf(abnf).map_err(|e| e.to_string())?;
+    let mut w = Vec::new();
+    abnf_to_pest::render_rules_to_pest(rules)
+      .render(0, &mut w)
+      .unwrap();
+    let pest = String::from_utf8(w).unwrap();
+
+    let pairs = pest_meta::parser::parse(pest_meta::parser::Rule::grammar_rules, &pest)
+      .map_err(|e| e.to_string())?;
+
+    let ast = pest_meta::parser::consume_rules(pairs).unwrap();
+
+    let vm = pest_vm::Vm::new(pest_meta::optimizer::optimize(ast.clone()));
+
+    let rule = rule.replace("-", "_");
+    let _ = vm.parse(&rule, target).map_err(|e| e.to_string())?;
+  }
+
+  Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use indoc::indoc;
+
+  #[test]
+  fn test_abnf() -> std::result::Result<(), Box<dyn std::error::Error>> {
+    let abnf_str = indoc!(
+      r#"
+        date-fullyear
+        date-fullyear   = 4DIGIT
+        date-month      = 2DIGIT  ; 01-12
+        date-mday       = 2DIGIT  ; 01-28, 01-29, 01-30, 01-31 based on
+                                  ; month/year
+        time-hour       = 2DIGIT  ; 00-23
+        time-minute     = 2DIGIT  ; 00-59
+        time-second     = 2DIGIT  ; 00-58, 00-59, 00-60 based on leap sec
+                                  ; rules
+        time-secfrac    = "." 1*DIGIT
+        time-numoffset  = ("+" / "-") time-hour ":" time-minute
+        time-offset     = "Z" / time-numoffset
+
+        partial-time    = time-hour ":" time-minute ":" time-second
+                          [time-secfrac]
+        full-date       = date-fullyear "-" date-month "-" date-mday
+        full-time       = partial-time time-offset
+
+        date-time       = full-date "T" full-time
+
+        DIGIT          =  %x30-39 ; 0-9
+        ; abbreviated here
+      "#
+    );
+
+    validate_abnf(abnf_str, "2009")?;
+
+    Ok(())
+  }
+}

src/validator/json.rs

@@ -6,6 +6,7 @@ use crate::{
   visitor::{self, *},
 };
 use chrono::{TimeZone, Utc};
+use control::validate_abnf;
 use serde_json::Value;
 use std::{borrow::Cow, collections::HashMap, convert::TryFrom, fmt};
 
@@ -1175,6 +1176,25 @@ impl<'a> Visitor<'a, Error> for JSONValidator<'a> {
 
         self.ctrl = None;
       }
+      t @ Some(Token::ABNF) => {
+        self.ctrl = t;
+
+        match target {
+          Type2::Typename { ident, .. } if is_ident_string_data_type(self.cddl, ident) => {
+            match self.json {
+              Value::String(_) | Value::Array(_) => self.visit_type2(controller)?,
+              _ => self.add_error(format!(
+                ".abnf control can only be matched against a JSON string, got {}",
+                self.json,
+              )),
+            }
+          }
+          _ => self.add_error(format!(
+            ".abnf can only be matched against string data type, got {}",
+            target,
+          )),
+        }
+      }
       _ => {
         self.add_error(format!("unsupported control operator {}", ctrl));
       }
@@ -2016,6 +2036,9 @@ impl<'a> Visitor<'a, Error> for JSONValidator<'a> {
               Some(format!("expected \"{}\" to match regex \"{}\"", s, t))
             }
           }
+          Some(Token::ABNF) => validate_abnf(t, s)
+            .err()
+            .map(|e| format!("\"{}\" is not valid against abnf: {}", s, e)),
           _ => {
             if s == t {
               None