add switch to change regex engine from Rust to Python (#983)

Author: davidhewitt

python/pydantic_core/core_schema.py

@@ -69,6 +69,7 @@ class CoreConfig(TypedDict, total=False):
         validation_error_cause: Whether to add user-python excs to the __cause__ of a ValidationError.
             Requires exceptiongroup backport pre Python 3.11.
         coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode).
+        regex_engine: The regex engine to use for regex pattern validation. Default is 'rust-regex'. See `StringSchema`.
     """
 
     title: str
@@ -752,6 +753,7 @@ class StringSchema(TypedDict, total=False):
     strip_whitespace: bool
     to_lower: bool
     to_upper: bool
+    regex_engine: Literal['rust-regex', 'python-re']  # default: 'rust-regex'
     strict: bool
     ref: str
     metadata: Any
@@ -766,6 +768,7 @@ def str_schema(
     strip_whitespace: bool | None = None,
     to_lower: bool | None = None,
     to_upper: bool | None = None,
+    regex_engine: Literal['rust-regex', 'python-re'] | None = None,
     strict: bool | None = None,
     ref: str | None = None,
     metadata: Any = None,
@@ -789,6 +792,12 @@ def str_schema(
         strip_whitespace: Whether to strip whitespace from the value
         to_lower: Whether to convert the value to lowercase
         to_upper: Whether to convert the value to uppercase
+        regex_engine: The regex engine to use for pattern validation. Default is 'rust-regex'.
+            - `rust-regex` uses the [`regex`](https://docs.rs/regex) Rust
+              crate, which is non-backtracking and therefore more DDoS
+              resistant, but does not support all regex features.
+            - `python-re` use the [`re`](https://docs.python.org/3/library/re.html) module,
+              which supports all regex features, but may be slower.
         strict: Whether the value should be a string or a value that can be converted to a string
         ref: optional unique identifier of the schema, used to reference the schema in other places
         metadata: Any other information you want to include with the schema, not used by pydantic-core
@@ -802,6 +811,7 @@ def str_schema(
         strip_whitespace=strip_whitespace,
         to_lower=to_lower,
         to_upper=to_upper,
+        regex_engine=regex_engine,
         strict=strict,
         ref=ref,
         metadata=metadata,

src/validators/string.rs

@@ -72,7 +72,7 @@ impl Validator for StrValidator {
 #[derive(Debug, Clone, Default)]
 pub struct StrConstrainedValidator {
     strict: bool,
-    pattern: Option<Regex>,
+    pattern: Option<Pattern>,
     max_length: Option<usize>,
     min_length: Option<usize>,
     strip_whitespace: bool,
@@ -126,10 +126,10 @@ impl Validator for StrConstrainedValidator {
         }
 
         if let Some(pattern) = &self.pattern {
-            if !pattern.is_match(str) {
+            if !pattern.is_match(py, str)? {
                 return Err(ValError::new(
                     ErrorType::StringPatternMismatch {
-                        pattern: pattern.to_string(),
+                        pattern: pattern.pattern.clone(),
                         context: None,
                     },
                     input,
@@ -170,10 +170,16 @@ impl Validator for StrConstrainedValidator {
 impl StrConstrainedValidator {
     fn build(schema: &PyDict, config: Option<&PyDict>) -> PyResult<Self> {
         let py = schema.py();
-        let pattern = match schema.get_as(intern!(py, "pattern"))? {
-            Some(s) => Some(Regex::new(s).map_err(|e| py_schema_error_type!("{}", e))?),
-            None => None,
-        };
+
+        let pattern = schema
+            .get_as(intern!(py, "pattern"))?
+            .map(|s| {
+                let regex_engine =
+                    schema_or_config(schema, config, intern!(py, "regex_engine"), intern!(py, "regex_engine"))?
+                        .unwrap_or(RegexEngine::RUST_REGEX);
+                Pattern::compile(py, s, regex_engine)
+            })
+            .transpose()?;
         let min_length: Option<usize> =
             schema_or_config(schema, config, intern!(py, "min_length"), intern!(py, "str_min_length"))?;
         let max_length: Option<usize> =
@@ -219,3 +225,45 @@ impl StrConstrainedValidator {
             || self.to_upper
     }
 }
+
+#[derive(Debug, Clone)]
+struct Pattern {
+    pattern: String,
+    engine: RegexEngine,
+}
+
+#[derive(Debug, Clone)]
+enum RegexEngine {
+    RustRegex(Regex),
+    PythonRe(PyObject),
+}
+
+impl RegexEngine {
+    const RUST_REGEX: &str = "rust-regex";
+    const PYTHON_RE: &str = "python-re";
+}
+
+impl Pattern {
+    fn compile(py: Python<'_>, pattern: String, engine: &str) -> PyResult<Self> {
+        let engine = match engine {
+            RegexEngine::RUST_REGEX => {
+                RegexEngine::RustRegex(Regex::new(&pattern).map_err(|e| py_schema_error_type!("{}", e))?)
+            }
+            RegexEngine::PYTHON_RE => {
+                let re_compile = py.import(intern!(py, "re"))?.getattr(intern!(py, "compile"))?;
+                RegexEngine::PythonRe(re_compile.call1((&pattern,))?.into())
+            }
+            _ => return Err(py_schema_error_type!("Invalid regex engine: {}", engine)),
+        };
+        Ok(Self { pattern, engine })
+    }
+
+    fn is_match(&self, py: Python<'_>, target: &str) -> PyResult<bool> {
+        match &self.engine {
+            RegexEngine::RustRegex(regex) => Ok(regex.is_match(target)),
+            RegexEngine::PythonRe(py_regex) => {
+                Ok(!py_regex.call_method1(py, intern!(py, "match"), (target,))?.is_none(py))
+            }
+        }
+    }
+}

tests/validators/test_string.py

@@ -167,26 +167,34 @@ def test_str_constrained_config():
         v.validate_python('test long')
 
 
-def test_invalid_regex():
+@pytest.mark.parametrize('engine', [None, 'rust-regex', 'python-re'])
+def test_invalid_regex(engine):
     # TODO uncomment and fix once #150 is done
     # with pytest.raises(SchemaError) as exc_info:
     #     SchemaValidator({'type': 'str', 'pattern': 123})
     # assert exc_info.value.args[0] == (
     #     'Error building "str" validator:\n  TypeError: \'int\' object cannot be converted to \'PyString\''
     # )
     with pytest.raises(SchemaError) as exc_info:
-        SchemaValidator({'type': 'str', 'pattern': '(abc'})
-    assert exc_info.value.args[0] == (
-        'Error building "str" validator:\n'
-        '  SchemaError: regex parse error:\n'
-        '    (abc\n'
-        '    ^\n'
-        'error: unclosed group'
-    )
-
-
-def test_regex_error():
-    v = SchemaValidator({'type': 'str', 'pattern': '11'})
+        SchemaValidator(core_schema.str_schema(pattern='(abc', regex_engine=engine))
+
+    if engine is None or engine == 'rust-regex':
+        assert exc_info.value.args[0] == (
+            'Error building "str" validator:\n'
+            '  SchemaError: regex parse error:\n'
+            '    (abc\n'
+            '    ^\n'
+            'error: unclosed group'
+        )
+    elif engine == 'python-re':
+        assert exc_info.value.args[0] == (
+            'Error building "str" validator:\n  error: missing ), unterminated subpattern at position 0'
+        )
+
+
+@pytest.mark.parametrize('engine', [None, 'rust-regex', 'python-re'])
+def test_regex_error(engine):
+    v = SchemaValidator(core_schema.str_schema(pattern='11', regex_engine=engine))
     with pytest.raises(ValidationError) as exc_info:
         v.validate_python('12')
     assert exc_info.value.errors(include_url=False) == [
@@ -297,3 +305,40 @@ def test_coerce_numbers_to_str_from_json(number: str, expected_str: str) -> None
 
     v = SchemaValidator(core_schema.str_schema(), config)
     assert v.validate_json(number) == expected_str
+
+
+@pytest.mark.parametrize('mode', (None, 'schema', 'config'))
+def test_backtracking_regex_rust_unsupported(mode) -> None:
+    pattern = r'r(#*)".*?"\1'
+
+    with pytest.raises(SchemaError) as exc_info:
+        if mode is None:
+            # rust-regex is the default
+            SchemaValidator(core_schema.str_schema(pattern=pattern))
+        elif mode == 'schema':
+            SchemaValidator(core_schema.str_schema(pattern=pattern, regex_engine='rust-regex'))
+        elif mode == 'config':
+            SchemaValidator(core_schema.str_schema(pattern=pattern), core_schema.CoreConfig(regex_engine='rust-regex'))
+
+    assert exc_info.value.args[0] == (
+        'Error building \"str\" validator:\n'
+        '  SchemaError: regex parse error:\n'
+        '    r(#*)\".*?\"\\1\n'
+        '              ^^\n'
+        'error: backreferences are not supported'
+    )
+
+
+@pytest.mark.parametrize('mode', ('schema', 'config'))
+def test_backtracking_regex_python(mode) -> None:
+    pattern = r'r(#*)".*?"\1'
+
+    if mode == 'schema':
+        v = SchemaValidator(core_schema.str_schema(pattern=pattern, regex_engine='python-re'))
+    elif mode == 'config':
+        v = SchemaValidator(core_schema.str_schema(pattern=pattern), core_schema.CoreConfig(regex_engine='python-re'))
+    assert v.validate_python('r""') == 'r""'
+    assert v.validate_python('r#""#') == 'r#""#'
+    with pytest.raises(ValidationError):
+        # not a valid match for the pattern
+        v.validate_python('r#"#')