Multi host URLs, other URL changes (#325)

* adding multi host urls

* linting and more test cases

* tweak unicode_string logic to replace range

* consume leading whitespace on multihost urls

* yet more test cases

* improve coverage

* add vulnerabilties tests

* remove "host_type" from url

* fix unicode positioning with multihost urls

* uprev

* allow default_host, default_port, default_path

* fix schema vs. scheme typo

* empty hosts() if no host

* tweaking url public methods

Author: samuelcolvin

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "pydantic-core"
-version = "0.7.0"
+version = "0.7.1"
 edition = "2021"
 license = "MIT"
 homepage = "https://github.com/pydantic/pydantic-core"

Cargo.toml

@@ -1,4 +1,5 @@
 from ._pydantic_core import (
+    MultiHostUrl,
     PydanticCustomError,
     PydanticKnownError,
     PydanticOmit,
@@ -16,6 +17,7 @@
     'CoreSchema',
     'SchemaValidator',
     'Url',
+    'MultiHostUrl',
     'SchemaError',
     'ValidationError',
     'PydanticCustomError',

pydantic_core/__init__.py

@@ -14,6 +14,7 @@ __all__ = (
     'build_profile',
     'SchemaValidator',
     'Url',
+    'MultiHostUrl',
     'SchemaError',
     'ValidationError',
     'PydanticCustomError',
@@ -44,19 +45,35 @@ class Url:
     username: 'str | None'
     password: 'str | None'
     host: 'str | None'
-    host_type: Literal['domain', 'punycode_domain', 'ipv4', 'ipv6', None]
     port: 'int | None'
     path: 'str | None'
     query: 'str | None'
     fragment: 'str | None'
 
-    def __init__(self, raw_url: str) -> None: ...
     def unicode_host(self) -> 'str | None': ...
     def query_params(self) -> 'list[tuple[str, str]]': ...
     def unicode_string(self) -> str: ...
     def __str__(self) -> str: ...
     def __repr__(self) -> str: ...
 
+class MultiHostHost(TypedDict):
+    username: 'str | None'
+    password: 'str | None'
+    host: str
+    port: 'int | None'
+
+class MultiHostUrl:
+    scheme: str
+    path: 'str | None'
+    query: 'str | None'
+    fragment: 'str | None'
+
+    def hosts(self) -> 'list[MultiHostHost]': ...
+    def query_params(self) -> 'list[tuple[str, str]]': ...
+    def unicode_string(self) -> str: ...
+    def __str__(self) -> str: ...
+    def __repr__(self) -> str: ...
+
 class SchemaError(Exception):
     pass
 

pydantic_core/_pydantic_core.pyi

@@ -1023,26 +1023,77 @@ def json_schema(schema: CoreSchema | None = None, *, ref: str | None = None, ext
 
 class UrlSchema(TypedDict, total=False):
     type: Required[Literal['url']]
-    host_required: bool  # default False
     max_length: int
     allowed_schemes: List[str]
+    host_required: bool  # default False
+    default_host: str
+    default_port: int
+    default_path: str
+    strict: bool
     ref: str
     extra: Any
 
 
 def url_schema(
     *,
-    host_required: bool | None = None,
     max_length: int | None = None,
     allowed_schemes: list[str] | None = None,
+    host_required: bool | None = None,
+    default_host: str | None = None,
+    default_port: int | None = None,
+    default_path: str | None = None,
+    strict: bool | None = None,
     ref: str | None = None,
     extra: Any = None,
 ) -> UrlSchema:
     return dict_not_none(
         type='url',
+        max_length=max_length,
+        allowed_schemes=allowed_schemes,
         host_required=host_required,
+        default_host=default_host,
+        default_port=default_port,
+        default_path=default_path,
+        strict=strict,
+        ref=ref,
+        extra=extra,
+    )
+
+
+class MultiHostUrlSchema(TypedDict, total=False):
+    type: Required[Literal['multi-host-url']]
+    max_length: int
+    allowed_schemes: List[str]
+    host_required: bool  # default False
+    default_host: str
+    default_port: int
+    default_path: str
+    strict: bool
+    ref: str
+    extra: Any
+
+
+def multi_host_url_schema(
+    *,
+    max_length: int | None = None,
+    allowed_schemes: list[str] | None = None,
+    host_required: bool | None = None,
+    default_host: str | None = None,
+    default_port: int | None = None,
+    default_path: str | None = None,
+    strict: bool | None = None,
+    ref: str | None = None,
+    extra: Any = None,
+) -> MultiHostUrlSchema:
+    return dict_not_none(
+        type='multi-host-url',
         max_length=max_length,
         allowed_schemes=allowed_schemes,
+        host_required=host_required,
+        default_host=default_host,
+        default_port=default_port,
+        default_path=default_path,
+        strict=strict,
         ref=ref,
         extra=extra,
     )
@@ -1087,6 +1138,7 @@ def url_schema(
     CustomErrorSchema,
     JsonSchema,
     UrlSchema,
+    MultiHostUrlSchema,
 ]
 
 # used in _pydantic_core.pyi::PydanticKnownError
@@ -1171,6 +1223,5 @@ def url_schema(
     'url_parsing',
     'url_syntax_violation',
     'url_too_long',
-    'url_schema',
-    'url_host_required',
+    'url_scheme',
 ]

pydantic_core/core_schema.py

@@ -334,12 +334,10 @@ pub enum ErrorType {
     UrlTooLong {
         max_length: usize,
     },
-    #[strum(message = "URL schema should be {expected_schemas}")]
-    UrlSchema {
-        expected_schemas: String,
+    #[strum(message = "URL scheme should be {expected_schemes}")]
+    UrlScheme {
+        expected_schemes: String,
     },
-    #[strum(message = "URL host required")]
-    UrlHostRequired,
 }
 
 macro_rules! render {
@@ -475,7 +473,7 @@ impl ErrorType {
             Self::UrlParsing { .. } => extract_context!(UrlParsing, ctx, error: String),
             Self::UrlSyntaxViolation { .. } => extract_context!(Cow::Owned, UrlSyntaxViolation, ctx, error: String),
             Self::UrlTooLong { .. } => extract_context!(UrlTooLong, ctx, max_length: usize),
-            Self::UrlSchema { .. } => extract_context!(UrlSchema, ctx, expected_schemas: String),
+            Self::UrlScheme { .. } => extract_context!(UrlScheme, ctx, expected_schemes: String),
             _ => {
                 if ctx.is_some() {
                     py_err!(PyTypeError; "'{}' errors do not require context", value)
@@ -566,7 +564,7 @@ impl ErrorType {
             Self::UrlParsing { error } => render!(self, error),
             Self::UrlSyntaxViolation { error } => render!(self, error),
             Self::UrlTooLong { max_length } => to_string_render!(self, max_length),
-            Self::UrlSchema { expected_schemas } => render!(self, expected_schemas),
+            Self::UrlScheme { expected_schemes } => render!(self, expected_schemes),
             _ => Ok(self.message_template().to_string()),
         }
     }
@@ -619,7 +617,7 @@ impl ErrorType {
             Self::UrlParsing { error } => py_dict!(py, error),
             Self::UrlSyntaxViolation { error } => py_dict!(py, error),
             Self::UrlTooLong { max_length } => py_dict!(py, max_length),
-            Self::UrlSchema { expected_schemas } => py_dict!(py, expected_schemas),
+            Self::UrlScheme { expected_schemes } => py_dict!(py, expected_schemes),
             _ => Ok(None),
         }
     }

src/errors/types.rs

@@ -4,7 +4,7 @@ use pyo3::prelude::*;
 use pyo3::types::{PyString, PyType};
 
 use crate::errors::{InputValue, LocItem, ValResult};
-use crate::PyUrl;
+use crate::{PyMultiHostUrl, PyUrl};
 
 use super::datetime::{EitherDate, EitherDateTime, EitherTime, EitherTimedelta};
 use super::return_enums::{EitherBytes, EitherString};
@@ -59,6 +59,10 @@ pub trait Input<'a>: fmt::Debug + ToPyObject {
         None
     }
 
+    fn input_as_multi_host_url(&self) -> Option<PyMultiHostUrl> {
+        None
+    }
+
     fn callable(&self) -> bool {
         false
     }

src/input/input_abstract.rs

@@ -13,7 +13,7 @@ use pyo3::types::{PyDictItems, PyDictKeys, PyDictValues};
 use pyo3::{ffi, intern, AsPyPointer, PyTypeInfo};
 
 use crate::errors::{py_err_string, ErrorType, InputValue, LocItem, ValError, ValLineError, ValResult};
-use crate::PyUrl;
+use crate::{PyMultiHostUrl, PyUrl};
 
 use super::datetime::{
     bytes_as_date, bytes_as_datetime, bytes_as_time, bytes_as_timedelta, date_as_datetime, float_as_datetime,
@@ -113,6 +113,10 @@ impl<'a> Input<'a> for PyAny {
         self.extract::<PyUrl>().ok()
     }
 
+    fn input_as_multi_host_url(&self) -> Option<PyMultiHostUrl> {
+        self.extract::<PyMultiHostUrl>().ok()
+    }
+
     fn callable(&self) -> bool {
         self.is_callable()
     }

src/input/input_python.rs

@@ -18,7 +18,7 @@ mod url;
 mod validators;
 
 // required for benchmarks
-pub use self::url::PyUrl;
+pub use self::url::{PyMultiHostUrl, PyUrl};
 pub use build_tools::SchemaError;
 pub use errors::{list_all_errors, PydanticCustomError, PydanticKnownError, PydanticOmit, ValidationError};
 pub use validators::SchemaValidator;
@@ -44,6 +44,7 @@ fn _pydantic_core(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<PydanticKnownError>()?;
     m.add_class::<PydanticOmit>()?;
     m.add_class::<self::url::PyUrl>()?;
+    m.add_class::<self::url::PyMultiHostUrl>()?;
     m.add_function(wrap_pyfunction!(list_all_errors, m)?)?;
     Ok(())
 }