From a1c61644cc2c974df95690a3716f4cc266045beb Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 10:30:24 -0500 Subject: [PATCH 01/18] Add __reduce__ method to SchemaSerializer Signed-off-by: Edward Oakes --- python/pydantic_core/_pydantic_core.pyi | 5 ++++- src/serializers/mod.rs | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index 8ed3092a9..9080317b8 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -2,7 +2,7 @@ from __future__ import annotations import datetime import sys -from typing import Any, Callable, Generic, Optional, Type, TypeVar +from typing import Any, Callable, Generic, Optional, Tuple, Type, TypeVar from pydantic_core import ErrorDetails, ErrorTypeInfo, InitErrorDetails, MultiHostHost from pydantic_core.core_schema import CoreConfig, CoreSchema, ErrorType @@ -263,6 +263,9 @@ class SchemaSerializer: schema: The [`CoreSchema`][pydantic_core.core_schema.CoreSchema] to use for serialization. config: Optionally a [`CoreConfig`][pydantic_core.core_schema.CoreConfig] to to configure serialization. """ + def __init__(self, schema: CoreSchema, config: CoreConfig | None = None) -> Self: + self._schema = schema + self._config = config def to_python( self, value: Any, diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index 72028346b..0bf7c7270 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -3,6 +3,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; +use pyo3::PyTypeInfo; use pyo3::{PyTraverseError, PyVisit}; use crate::definitions::{Definitions, DefinitionsBuilder}; @@ -33,6 +34,10 @@ pub struct SchemaSerializer { definitions: Definitions, expected_json_size: AtomicUsize, config: SerializationConfig, + // References to the Python schema and config objects are saved to enable + // reconstructing the object for cloudpickle support (see `__reduce__`). + py_schema: Py, + py_config: Option>, } impl SchemaSerializer { @@ -71,15 +76,19 @@ impl SchemaSerializer { #[pymethods] impl SchemaSerializer { #[new] - pub fn py_new(schema: &PyDict, config: Option<&PyDict>) -> PyResult { + pub fn py_new(py: Python, schema: &PyDict, config: Option<&PyDict>) -> PyResult { let mut definitions_builder = DefinitionsBuilder::new(); - let serializer = CombinedSerializer::build(schema.downcast()?, config, &mut definitions_builder)?; Ok(Self { serializer, definitions: definitions_builder.finish()?, expected_json_size: AtomicUsize::new(1024), config: SerializationConfig::from_config(config)?, + py_schema: schema.into_py(py), + py_config: match config { + Some(d) if !d.is_empty() => Some(d.into_py(py)), + _ => None, + }, }) } @@ -174,6 +183,14 @@ impl SchemaSerializer { Ok(py_bytes.into()) } + pub fn __reduce__(&self, py: Python) -> PyResult<(PyObject, (PyObject, PyObject))> { + // Enables support for `cloudpickle` serialization. + Ok(( + SchemaSerializer::type_object(py).to_object(py), + (self.py_schema.to_object(py), self.py_config.to_object(py)), + )) + } + pub fn __repr__(&self) -> String { format!( "SchemaSerializer(serializer={:#?}, definitions={:#?})", From ce155b87862b5fa2256f2e2f6720a9f498b58117 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 10:36:04 -0500 Subject: [PATCH 02/18] Revert changes to _pydantic_core.pyi Signed-off-by: Edward Oakes --- python/pydantic_core/_pydantic_core.pyi | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index 9080317b8..8ed3092a9 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -2,7 +2,7 @@ from __future__ import annotations import datetime import sys -from typing import Any, Callable, Generic, Optional, Tuple, Type, TypeVar +from typing import Any, Callable, Generic, Optional, Type, TypeVar from pydantic_core import ErrorDetails, ErrorTypeInfo, InitErrorDetails, MultiHostHost from pydantic_core.core_schema import CoreConfig, CoreSchema, ErrorType @@ -263,9 +263,6 @@ class SchemaSerializer: schema: The [`CoreSchema`][pydantic_core.core_schema.CoreSchema] to use for serialization. config: Optionally a [`CoreConfig`][pydantic_core.core_schema.CoreConfig] to to configure serialization. """ - def __init__(self, schema: CoreSchema, config: CoreConfig | None = None) -> Self: - self._schema = schema - self._config = config def to_python( self, value: Any, From 315908456cfa9c518959813d00bb96b6ab965ff1 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 10:57:16 -0500 Subject: [PATCH 03/18] Fix constructor in tests Signed-off-by: Edward Oakes --- src/serializers/mod.rs | 2 ++ tests/test.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index 0bf7c7270..edf8c6201 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -199,6 +199,8 @@ impl SchemaSerializer { } fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.py_schema)?; + visit.call(&self.py_config)?; self.serializer.py_gc_traverse(&visit)?; self.definitions.py_gc_traverse(&visit)?; Ok(()) diff --git a/tests/test.rs b/tests/test.rs index 526b30e5e..9b2fb99b5 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -46,7 +46,7 @@ mod tests { ] }"#; let schema: &PyDict = py.eval(code, None, None).unwrap().extract().unwrap(); - SchemaSerializer::py_new(schema, None).unwrap(); + SchemaSerializer::py_new(py, schema, None).unwrap(); }); } @@ -77,7 +77,7 @@ a = A() py.run(code, None, Some(locals)).unwrap(); let a: &PyAny = locals.get_item("a").unwrap().extract().unwrap(); let schema: &PyDict = locals.get_item("schema").unwrap().extract().unwrap(); - let serialized: Vec = SchemaSerializer::py_new(schema, None) + let serialized: Vec = SchemaSerializer::py_new(py, schema, None) .unwrap() .to_json(py, a, None, None, None, true, false, false, false, false, true, None) .unwrap() From cdc32a29067936a7939d7d34a5f89a940f7a49d0 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 11:03:32 -0500 Subject: [PATCH 04/18] Handle Option in __traverse__ Signed-off-by: Edward Oakes --- src/serializers/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index edf8c6201..9e2d358f6 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -200,7 +200,9 @@ impl SchemaSerializer { fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.py_schema)?; - visit.call(&self.py_config)?; + if let Some(ref py_config) = self.py_config { + visit.call(py_config)?; + } self.serializer.py_gc_traverse(&visit)?; self.definitions.py_gc_traverse(&visit)?; Ok(()) From bba95d88f4aa60f0086e28cb2383edd0cfda4cb0 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 11:20:59 -0500 Subject: [PATCH 05/18] Make the linter happy Signed-off-by: Edward Oakes --- src/validators/url.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validators/url.rs b/src/validators/url.rs index 4584ae652..b5fe8bf4c 100644 --- a/src/validators/url.rs +++ b/src/validators/url.rs @@ -498,7 +498,7 @@ fn check_sub_defaults( if let Some(default_port) = default_port { lib_url .set_port(Some(default_port)) - .map_err(|_| map_parse_err(ParseError::EmptyHost))?; + .map_err(|()| map_parse_err(ParseError::EmptyHost))?; } } if let Some(ref default_path) = default_path { From 24ad5b085c96d22a2027e7a1213191fdfb486321 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 11:54:12 -0500 Subject: [PATCH 06/18] Add test_pickling.py Signed-off-by: Edward Oakes --- tests/test_pickling.py | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/test_pickling.py diff --git a/tests/test_pickling.py b/tests/test_pickling.py new file mode 100644 index 000000000..c59dafade --- /dev/null +++ b/tests/test_pickling.py @@ -0,0 +1,60 @@ +import copy +import json +import pickle +import re +from datetime import timedelta + +import pytest +from typing_extensions import get_args + +from pydantic_core import CoreSchema, CoreSchemaType, PydanticUndefined, core_schema +from pydantic_core._pydantic_core import ( + SchemaError, + SchemaSerializer, + ValidationError, + __version__, + build_info, + build_profile, +) + + +def repr_function(value, _info): + return repr(value) + + +def test_basic_schema_serializer(): + s = SchemaSerializer(core_schema.dict_schema()) + s = pickle.loads(pickle.dumps(s)) + assert s.to_python({'a': 1, b'b': 2, 33: 3}) == {'a': 1, b'b': 2, 33: 3} + assert s.to_python({'a': 1, b'b': 2, 33: 3, True: 4}, mode='json') == {'a': 1, 'b': 2, '33': 3, 'true': 4} + assert s.to_json({'a': 1, b'b': 2, 33: 3, True: 4}) == b'{"a":1,"b":2,"33":3,"true":4}' + + assert s.to_python({(1, 2): 3}) == {(1, 2): 3} + assert s.to_python({(1, 2): 3}, mode='json') == {'1,2': 3} + assert s.to_json({(1, 2): 3}) == b'{"1,2":3}' + + +@pytest.mark.parametrize( + 'value,expected_python,expected_json', + [(None, 'None', b'"None"'), (1, '1', b'"1"'), ([1, 2, 3], '[1, 2, 3]', b'"[1, 2, 3]"')], +) +def test_schema_serializer_capturing_function(value, expected_python, expected_json): + # Test a SchemaSerializer that captures a function. + s = SchemaSerializer( + core_schema.any_schema( + serialization=core_schema.plain_serializer_function_ser_schema(repr_function, info_arg=True) + ) + ) + s = pickle.loads(pickle.dumps(s)) + assert s.to_python(value) == expected_python + assert s.to_json(value) == expected_json + assert s.to_python(value, mode='json') == json.loads(expected_json) + + +def test_schema_serializer_containing_config(): + s = SchemaSerializer(core_schema.timedelta_schema(), config={'ser_json_timedelta': 'float'}) + s = pickle.loads(pickle.dumps(s)) + + assert s.to_python(timedelta(seconds=4, microseconds=500_000)) == timedelta(seconds=4, microseconds=500_000) + assert s.to_python(timedelta(seconds=4, microseconds=500_000), mode='json') == 4.5 + assert s.to_json(timedelta(seconds=4, microseconds=500_000)) == b'4.5' From bdafefdc4fb187e9284c53b0d7add890dc841260 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 11:56:07 -0500 Subject: [PATCH 07/18] Fix imports Signed-off-by: Edward Oakes --- tests/test_pickling.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_pickling.py b/tests/test_pickling.py index c59dafade..770b6d2c2 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -5,16 +5,10 @@ from datetime import timedelta import pytest -from typing_extensions import get_args -from pydantic_core import CoreSchema, CoreSchemaType, PydanticUndefined, core_schema +from pydantic_core import core_schema from pydantic_core._pydantic_core import ( - SchemaError, SchemaSerializer, - ValidationError, - __version__, - build_info, - build_profile, ) From 8c41c88153a4bb5aad320c3415a8ea4e5c853bf1 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 11:59:59 -0500 Subject: [PATCH 08/18] Fix imports Signed-off-by: Edward Oakes --- tests/test_pickling.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_pickling.py b/tests/test_pickling.py index 770b6d2c2..e6e87cd9c 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -1,7 +1,5 @@ -import copy import json import pickle -import re from datetime import timedelta import pytest From 70f303200fa91d479e7bb7d4f54d8a9eca634bf4 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 12:10:09 -0500 Subject: [PATCH 09/18] Fix lint Signed-off-by: Edward Oakes --- tests/test_pickling.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_pickling.py b/tests/test_pickling.py index e6e87cd9c..2ca230313 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -5,9 +5,7 @@ import pytest from pydantic_core import core_schema -from pydantic_core._pydantic_core import ( - SchemaSerializer, -) +from pydantic_core._pydantic_core import SchemaSerializer def repr_function(value, _info): From 8b8bfd5fe2a7c104a64442e179ef408daed87eb0 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 12:55:49 -0500 Subject: [PATCH 10/18] Add config to gc.collect() test Signed-off-by: Edward Oakes --- tests/test_garbage_collection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_garbage_collection.py b/tests/test_garbage_collection.py index d848c91ea..d5d710407 100644 --- a/tests/test_garbage_collection.py +++ b/tests/test_garbage_collection.py @@ -27,7 +27,10 @@ class BaseModel: __schema__: SchemaSerializer def __init_subclass__(cls) -> None: - cls.__schema__ = SchemaSerializer(core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER)) + cls.__schema__ = SchemaSerializer( + core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), + config={'ser_json_timedelta': 'float'} + ) cache: 'WeakValueDictionary[int, Any]' = WeakValueDictionary() From 661ac07ec3bdea93d1fb07ac20da3d576f92ff1d Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 6 Oct 2023 12:59:12 -0500 Subject: [PATCH 11/18] Fix lint Signed-off-by: Edward Oakes --- tests/test_garbage_collection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_garbage_collection.py b/tests/test_garbage_collection.py index d5d710407..798b66bbe 100644 --- a/tests/test_garbage_collection.py +++ b/tests/test_garbage_collection.py @@ -28,8 +28,7 @@ class BaseModel: def __init_subclass__(cls) -> None: cls.__schema__ = SchemaSerializer( - core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), - config={'ser_json_timedelta': 'float'} + core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), config={'ser_json_timedelta': 'float'} ) cache: 'WeakValueDictionary[int, Any]' = WeakValueDictionary() From 3e9b372e3e67b0473c6bb8c5ac82a5fffa1ed3c8 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 10:00:06 -0500 Subject: [PATCH 12/18] Use slf.get_type() Signed-off-by: Edward Oakes --- src/serializers/mod.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index 9e2d358f6..ddea120d9 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -3,7 +3,6 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; -use pyo3::PyTypeInfo; use pyo3::{PyTraverseError, PyVisit}; use crate::definitions::{Definitions, DefinitionsBuilder}; @@ -27,7 +26,7 @@ mod ob_type; mod shared; mod type_serializers; -#[pyclass(module = "pydantic_core._pydantic_core")] +#[pyclass(module = "pydantic_core._pydantic_core", frozen)] #[derive(Debug)] pub struct SchemaSerializer { serializer: CombinedSerializer, @@ -183,12 +182,12 @@ impl SchemaSerializer { Ok(py_bytes.into()) } - pub fn __reduce__(&self, py: Python) -> PyResult<(PyObject, (PyObject, PyObject))> { + pub fn __reduce__(slf: &PyCell) -> PyResult<(PyObject, (PyObject, PyObject))> { // Enables support for `cloudpickle` serialization. - Ok(( - SchemaSerializer::type_object(py).to_object(py), - (self.py_schema.to_object(py), self.py_config.to_object(py)), - )) + let py = slf.py(); + let cls = slf.get_type().into(); + let init_args = (slf.get().py_schema.to_object(py), slf.get().py_config.to_object(py)); + Ok((cls, init_args)) } pub fn __repr__(&self) -> String { From 9ef470b3e834818d5702252fd355103514350674 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 10:34:41 -0500 Subject: [PATCH 13/18] Update validator __reduce__ to include config Signed-off-by: Edward Oakes --- src/serializers/mod.rs | 6 +-- src/validators/mod.rs | 44 ++++++++++------ tests/{ => serializers}/test_pickling.py | 0 tests/validators/test_datetime.py | 12 ----- tests/validators/test_pickling.py | 65 ++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 30 deletions(-) rename tests/{ => serializers}/test_pickling.py (100%) create mode 100644 tests/validators/test_pickling.py diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index ddea120d9..00d70162f 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -34,7 +34,7 @@ pub struct SchemaSerializer { expected_json_size: AtomicUsize, config: SerializationConfig, // References to the Python schema and config objects are saved to enable - // reconstructing the object for cloudpickle support (see `__reduce__`). + // reconstructing the object for pickle support (see `__reduce__`). py_schema: Py, py_config: Option>, } @@ -85,7 +85,7 @@ impl SchemaSerializer { config: SerializationConfig::from_config(config)?, py_schema: schema.into_py(py), py_config: match config { - Some(d) if !d.is_empty() => Some(d.into_py(py)), + Some(c) if !c.is_empty() => Some(c.into_py(py)), _ => None, }, }) @@ -183,7 +183,7 @@ impl SchemaSerializer { } pub fn __reduce__(slf: &PyCell) -> PyResult<(PyObject, (PyObject, PyObject))> { - // Enables support for `cloudpickle` serialization. + // Enables support for `pickle` serialization. let py = slf.py(); let cls = slf.get_type().into(); let init_args = (slf.get().py_schema.to_object(py), slf.get().py_config.to_object(py)); diff --git a/src/validators/mod.rs b/src/validators/mod.rs index 42aad2001..48f3ef837 100644 --- a/src/validators/mod.rs +++ b/src/validators/mod.rs @@ -97,14 +97,17 @@ impl PySome { } } -#[pyclass(module = "pydantic_core._pydantic_core")] +#[pyclass(module = "pydantic_core._pydantic_core", frozen)] #[derive(Debug)] pub struct SchemaValidator { validator: CombinedValidator, definitions: Definitions, - schema: PyObject, + // References to the Python schema and config objects are saved to enable + // reconstructing the object for cloudpickle support (see `__reduce__`). + py_schema: Py, + py_config: Option>, #[pyo3(get)] - title: PyObject, + py_title: Py, hide_input_in_errors: bool, validation_error_cause: bool, } @@ -121,11 +124,16 @@ impl SchemaValidator { for val in definitions.values() { val.get().unwrap().complete()?; } + let py_schema = schema.into_py(py); + let py_config = match config { + Some(c) if !c.is_empty() => Some(c.into_py(py)), + _ => None, + }; let config_title = match config { Some(c) => c.get_item("title"), None => None, }; - let title = match config_title { + let py_title = match config_title { Some(t) => t.into_py(py), None => validator.get_name().into_py(py), }; @@ -134,18 +142,20 @@ impl SchemaValidator { Ok(Self { validator, definitions, - schema: schema.into_py(py), - title, + py_schema, + py_config, + py_title, hide_input_in_errors, validation_error_cause, }) } - pub fn __reduce__(slf: &PyCell) -> PyResult { + pub fn __reduce__(slf: &PyCell) -> PyResult<(PyObject, (PyObject, PyObject))> { + // Enables support for `pickle` serialization. let py = slf.py(); - let args = (slf.try_borrow()?.schema.to_object(py),); - let cls = slf.getattr("__class__")?; - Ok((cls, args).into_py(py)) + let cls = slf.get_type().into(); + let init_args = (slf.get().py_schema.to_object(py), slf.get().py_config.to_object(py)); + Ok((cls, init_args)) } #[pyo3(signature = (input, *, strict=None, from_attributes=None, context=None, self_instance=None))] @@ -299,7 +309,7 @@ impl SchemaValidator { pub fn __repr__(&self, py: Python) -> String { format!( "SchemaValidator(title={:?}, validator={:#?}, definitions={:#?})", - self.title.extract::<&str>(py).unwrap(), + self.py_title.extract::<&str>(py).unwrap(), self.validator, self.definitions, ) @@ -307,7 +317,10 @@ impl SchemaValidator { fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { self.validator.py_gc_traverse(&visit)?; - visit.call(&self.schema)?; + visit.call(&self.py_schema)?; + if let Some(ref py_config) = self.py_config { + visit.call(py_config)?; + } Ok(()) } } @@ -338,7 +351,7 @@ impl SchemaValidator { fn prepare_validation_err(&self, py: Python, error: ValError, input_type: InputType) -> PyErr { ValidationError::from_val_error( py, - self.title.clone_ref(py), + self.py_title.clone_ref(py), input_type, error, None, @@ -396,8 +409,9 @@ impl<'py> SelfValidator<'py> { Ok(SchemaValidator { validator, definitions, - schema: py.None(), - title: "Self Schema".into_py(py), + py_schema: py.None(), + py_config: None, + py_title: "Self Schema".into_py(py), hide_input_in_errors: false, validation_error_cause: false, }) diff --git a/tests/test_pickling.py b/tests/serializers/test_pickling.py similarity index 100% rename from tests/test_pickling.py rename to tests/serializers/test_pickling.py diff --git a/tests/validators/test_datetime.py b/tests/validators/test_datetime.py index df04d1631..67581119b 100644 --- a/tests/validators/test_datetime.py +++ b/tests/validators/test_datetime.py @@ -1,6 +1,5 @@ import copy import json -import pickle import platform import re from datetime import date, datetime, time, timedelta, timezone, tzinfo @@ -480,17 +479,6 @@ def test_tz_constraint_wrong(): validate_core_schema(core_schema.datetime_schema(tz_constraint='wrong')) -def test_tz_pickle() -> None: - """ - https://github.com/pydantic/pydantic-core/issues/589 - """ - v = SchemaValidator(core_schema.datetime_schema()) - original = datetime(2022, 6, 8, 12, 13, 14, tzinfo=timezone(timedelta(hours=-12, minutes=-15))) - validated = v.validate_python('2022-06-08T12:13:14-12:15') - assert validated == original - assert pickle.loads(pickle.dumps(validated)) == validated == original - - def test_tz_hash() -> None: v = SchemaValidator(core_schema.datetime_schema()) lookup: Dict[datetime, str] = {} diff --git a/tests/validators/test_pickling.py b/tests/validators/test_pickling.py new file mode 100644 index 000000000..c3ffa0f0b --- /dev/null +++ b/tests/validators/test_pickling.py @@ -0,0 +1,65 @@ +import dataclasses +import pickle +import re +from datetime import datetime, timedelta, timezone + +import pytest + +from pydantic_core import core_schema +from pydantic_core._pydantic_core import SchemaValidator, ValidationError + +from ..conftest import PyAndJson + + +def test_basic_schema_validator(py_and_json: PyAndJson): + v = py_and_json({'type': 'dict', 'keys_schema': {'type': 'int'}, 'values_schema': {'type': 'int'}}) + v = pickle.loads(pickle.dumps(v)) + assert v.validate_test({'1': 2, '3': 4}) == {1: 2, 3: 4} + + v = py_and_json({'type': 'dict', 'strict': True, 'keys_schema': {'type': 'int'}, 'values_schema': {'type': 'int'}}) + v = pickle.loads(pickle.dumps(v)) + assert v.validate_test({'1': 2, '3': 4}) == {1: 2, 3: 4} + assert v.validate_test({}) == {} + with pytest.raises(ValidationError, match=re.escape('[type=dict_type, input_value=[], input_type=list]')): + v.validate_test([]) + + +def test_schema_validator_containing_config(): + """ + Verify that the config object is not lost during (de)serialization. + """ + + @dataclasses.dataclass + class MyModel: + f: str + + v = SchemaValidator( + core_schema.dataclass_schema( + MyModel, + core_schema.dataclass_args_schema('MyModel', [core_schema.dataclass_field('f', core_schema.str_schema())]), + ['f'], + config=core_schema.CoreConfig(extra_fields_behavior='allow'), + ) + ) + + m: MyModel = v.validate_python({'f': 'x', 'extra_field': '123'}) + assert m.f == 'x' + assert getattr(m, 'extra_field') == '123' + + # If the config was lost during (de)serialization, the validation call below would + # fail due to the `extra_field`. + v = pickle.loads(pickle.dumps(v)) + m: MyModel = v.validate_python({'f': 'x', 'extra_field': '123'}) + assert m.f == 'x' + assert getattr(m, 'extra_field') == '123' + + +def test_schema_validator_tz_pickle() -> None: + """ + https://github.com/pydantic/pydantic-core/issues/589 + """ + v = SchemaValidator(core_schema.datetime_schema()) + original = datetime(2022, 6, 8, 12, 13, 14, tzinfo=timezone(timedelta(hours=-12, minutes=-15))) + validated = v.validate_python('2022-06-08T12:13:14-12:15') + assert validated == original + assert pickle.loads(pickle.dumps(validated)) == validated == original From 26cb67a62cc35838ca2b8fca6608e3a20c07530f Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 10:57:40 -0500 Subject: [PATCH 14/18] Fix config test Signed-off-by: Edward Oakes --- tests/validators/test_pickling.py | 32 +++++++++++-------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/tests/validators/test_pickling.py b/tests/validators/test_pickling.py index c3ffa0f0b..37fda2263 100644 --- a/tests/validators/test_pickling.py +++ b/tests/validators/test_pickling.py @@ -1,4 +1,3 @@ -import dataclasses import pickle import re from datetime import datetime, timedelta, timezone @@ -28,30 +27,21 @@ def test_schema_validator_containing_config(): """ Verify that the config object is not lost during (de)serialization. """ - - @dataclasses.dataclass - class MyModel: - f: str - v = SchemaValidator( - core_schema.dataclass_schema( - MyModel, - core_schema.dataclass_args_schema('MyModel', [core_schema.dataclass_field('f', core_schema.str_schema())]), - ['f'], - config=core_schema.CoreConfig(extra_fields_behavior='allow'), - ) + core_schema.model_fields_schema({'f': core_schema.model_field(core_schema.str_schema())}), + config=core_schema.CoreConfig(extra_fields_behavior='allow'), ) + v = pickle.loads(pickle.dumps(v)) - m: MyModel = v.validate_python({'f': 'x', 'extra_field': '123'}) - assert m.f == 'x' - assert getattr(m, 'extra_field') == '123' + m, model_extra, fields_set = v.validate_python({'f': 'x', 'extra_field': '123'}) + assert m == {'f': 'x'} + # If the config was lost during (de)serialization, the below checks would fail as + # the default behavior is to ignore extra fields. + assert model_extra == {'extra_field': '123'} + assert fields_set == {'f', 'extra_field'} - # If the config was lost during (de)serialization, the validation call below would - # fail due to the `extra_field`. - v = pickle.loads(pickle.dumps(v)) - m: MyModel = v.validate_python({'f': 'x', 'extra_field': '123'}) - assert m.f == 'x' - assert getattr(m, 'extra_field') == '123' + v.validate_assignment(m, 'f', 'y') + assert m == {'f': 'y'} def test_schema_validator_tz_pickle() -> None: From ed3ad18a4a927bc0ef18c89ce40e9c2e8828b5bd Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 11:07:47 -0500 Subject: [PATCH 15/18] Fix tests Signed-off-by: Edward Oakes --- src/validators/mod.rs | 12 ++++++------ tests/validators/test_pickling.py | 22 ++++++++++------------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/validators/mod.rs b/src/validators/mod.rs index 48f3ef837..4ff57e319 100644 --- a/src/validators/mod.rs +++ b/src/validators/mod.rs @@ -107,7 +107,7 @@ pub struct SchemaValidator { py_schema: Py, py_config: Option>, #[pyo3(get)] - py_title: Py, + title: Py, hide_input_in_errors: bool, validation_error_cause: bool, } @@ -133,7 +133,7 @@ impl SchemaValidator { Some(c) => c.get_item("title"), None => None, }; - let py_title = match config_title { + let title = match config_title { Some(t) => t.into_py(py), None => validator.get_name().into_py(py), }; @@ -144,7 +144,7 @@ impl SchemaValidator { definitions, py_schema, py_config, - py_title, + title, hide_input_in_errors, validation_error_cause, }) @@ -309,7 +309,7 @@ impl SchemaValidator { pub fn __repr__(&self, py: Python) -> String { format!( "SchemaValidator(title={:?}, validator={:#?}, definitions={:#?})", - self.py_title.extract::<&str>(py).unwrap(), + self.title.extract::<&str>(py).unwrap(), self.validator, self.definitions, ) @@ -351,7 +351,7 @@ impl SchemaValidator { fn prepare_validation_err(&self, py: Python, error: ValError, input_type: InputType) -> PyErr { ValidationError::from_val_error( py, - self.py_title.clone_ref(py), + self.title.clone_ref(py), input_type, error, None, @@ -411,7 +411,7 @@ impl<'py> SelfValidator<'py> { definitions, py_schema: py.None(), py_config: None, - py_title: "Self Schema".into_py(py), + title: "Self Schema".into_py(py), hide_input_in_errors: false, validation_error_cause: false, }) diff --git a/tests/validators/test_pickling.py b/tests/validators/test_pickling.py index 37fda2263..2037ab8c9 100644 --- a/tests/validators/test_pickling.py +++ b/tests/validators/test_pickling.py @@ -4,23 +4,21 @@ import pytest -from pydantic_core import core_schema +from pydantic_core import core_schema, validate_core_schema from pydantic_core._pydantic_core import SchemaValidator, ValidationError -from ..conftest import PyAndJson - -def test_basic_schema_validator(py_and_json: PyAndJson): - v = py_and_json({'type': 'dict', 'keys_schema': {'type': 'int'}, 'values_schema': {'type': 'int'}}) - v = pickle.loads(pickle.dumps(v)) - assert v.validate_test({'1': 2, '3': 4}) == {1: 2, 3: 4} - - v = py_and_json({'type': 'dict', 'strict': True, 'keys_schema': {'type': 'int'}, 'values_schema': {'type': 'int'}}) +def test_basic_schema_validator(): + v = SchemaValidator( + validate_core_schema( + {'type': 'dict', 'strict': True, 'keys_schema': {'type': 'int'}, 'values_schema': {'type': 'int'}} + ) + ) v = pickle.loads(pickle.dumps(v)) - assert v.validate_test({'1': 2, '3': 4}) == {1: 2, 3: 4} - assert v.validate_test({}) == {} + assert v.validate_python({'1': 2, '3': 4}) == {1: 2, 3: 4} + assert v.validate_python({}) == {} with pytest.raises(ValidationError, match=re.escape('[type=dict_type, input_value=[], input_type=list]')): - v.validate_test([]) + v.validate_python([]) def test_schema_validator_containing_config(): From e11ba19a0e6afccf26ec689de94cb8937513ecc2 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 11:25:53 -0500 Subject: [PATCH 16/18] Revert small type change Signed-off-by: Edward Oakes --- src/validators/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validators/mod.rs b/src/validators/mod.rs index 4ff57e319..5db192f3a 100644 --- a/src/validators/mod.rs +++ b/src/validators/mod.rs @@ -107,7 +107,7 @@ pub struct SchemaValidator { py_schema: Py, py_config: Option>, #[pyo3(get)] - title: Py, + title: PyObject, hide_input_in_errors: bool, validation_error_cause: bool, } From 1d17408dbe54690c0d42fd5bc86ddfa376450df6 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 11:31:13 -0500 Subject: [PATCH 17/18] Add config to gc test Signed-off-by: Edward Oakes --- tests/test_garbage_collection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_garbage_collection.py b/tests/test_garbage_collection.py index 798b66bbe..f5531e172 100644 --- a/tests/test_garbage_collection.py +++ b/tests/test_garbage_collection.py @@ -58,7 +58,10 @@ class BaseModel: __validator__: SchemaValidator def __init_subclass__(cls) -> None: - cls.__validator__ = SchemaValidator(core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER)) + cls.__validator__ = SchemaValidator( + core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), + core_schema.CoreConfig(extra_fields_behavior='allow'), + ) cache: 'WeakValueDictionary[int, Any]' = WeakValueDictionary() From 722bbdfa1dd2b8b767f84395db5a3f1c327a46de Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 9 Oct 2023 11:32:42 -0500 Subject: [PATCH 18/18] small nit Signed-off-by: Edward Oakes --- tests/test_garbage_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_garbage_collection.py b/tests/test_garbage_collection.py index f5531e172..97107e61b 100644 --- a/tests/test_garbage_collection.py +++ b/tests/test_garbage_collection.py @@ -60,7 +60,7 @@ class BaseModel: def __init_subclass__(cls) -> None: cls.__validator__ = SchemaValidator( core_schema.model_schema(cls, GC_TEST_SCHEMA_INNER), - core_schema.CoreConfig(extra_fields_behavior='allow'), + config=core_schema.CoreConfig(extra_fields_behavior='allow'), ) cache: 'WeakValueDictionary[int, Any]' = WeakValueDictionary()