@handle_urls() with item return type

.flake8

@@ -34,5 +34,7 @@ per-file-ignores =
     # imports are there to expose submodule functions so they can be imported
     # directly from that module
     # F403: Ignore * imports in these files
+    # D102: Missing docstring in public method
     web_poet/__init__.py:F401,F403
     web_poet/page_inputs/__init__.py:F401,F403
+    tests/po_lib_to_return/__init__.py:D102 

CHANGELOG.rst

@@ -2,6 +2,37 @@
 Changelog
 =========
 
+TBD
+---
+
+* New ``ApplyRule`` class created by the ``@handle_urls`` decorator.
+  This is nearly identical with ``OverrideRule`` except that it's accepting
+  a ``to_return`` parameter which signifies the data container class that
+  the Page Object returns.
+
+* Modify the call signature of ``handle_urls``:
+
+    * New ``to_return`` parameter which signifies the data container class that
+      the Page Object returns. This behaves exactly the same with ``overrides``
+      but it's more consistent with  the attribute names of ``ApplyRules`` that
+      it creates.
+    * New ``instead_of`` parameter which does the same thing with ``overrides``.
+    * The old ``overrides`` parameter is not required anymore as it's set for
+      deprecation.
+
+Deprecations:
+
+* The ``overrides`` parameter from ``@handle_urls`` is now deprecated.
+  Use the ``instead_of`` parameter instead.
+* ``OverrideRule`` is now deprecated. Use ``ApplyRule`` instead.
+* The ``from_override_rules`` method of ``PageObjectRegistry`` is now deprecated.
+  Use ``from_apply_rules`` instead.
+* The ``web_poet.overrides`` is deprecated. Use ``web_poet.rules`` instead.
+* The ``PageObjectRegistry.get_overrides`` is deprecated.
+  Use ``PageObjectRegistry.get_rules`` instead.
+* The ``PageObjectRegistry.search_overrides`` is deprecated.
+  Use ``PageObjectRegistry.search_rules`` instead.
+
 0.5.1 (2022-09-23)
 ------------------
 

docs/advanced/fields.rst

@@ -319,7 +319,7 @@ inherit from the "base", "standard" Page Object, there could be a ``@field``
 from the base class which is not present in the ``CustomItem``.
 It'd be still passed to ``CustomItem.__init__``, causing an exception.
 
-One way to solve it is to make the orignal Page Object a dependency
+One way to solve it is to make the original Page Object a dependency
 instead of inheriting from it, as explained in the beginning.
 
 Alternatively, you can use ``skip_nonitem_fields=True`` class argument - it tells

docs/api-reference.rst

@@ -91,7 +91,7 @@ use cases and some examples.
 
 .. autofunction:: web_poet.handle_urls
 
-.. automodule:: web_poet.overrides
+.. automodule:: web_poet.rules
    :members:
    :exclude-members: handle_urls
 

pyproject.toml

@@ -6,3 +6,7 @@ multi_line_output = 3
 show_error_codes = true
 ignore_missing_imports = true
 no_warn_no_return = true
+
+[[tool.mypy.overrides]]
+module = "tests.po_lib_to_return.*"
+ignore_errors = true

tests/po_lib/__init__.py

@@ -12,8 +12,9 @@
 
 
 class POBase(ItemPage):
-    expected_overrides: Type[ItemPage]
+    expected_instead_of: Type[ItemPage]
     expected_patterns: Patterns
+    expected_to_return = None
     expected_meta: Dict[str, Any]
 
 
@@ -26,18 +27,20 @@ class POTopLevelOverriden2(ItemPage):
 
 
 # This first annotation is ignored. A single annotation per registry is allowed
-@handle_urls("example.com", overrides=POTopLevelOverriden1)
+@handle_urls("example.com", instead_of=POTopLevelOverriden1)
 @handle_urls(
-    "example.com", overrides=POTopLevelOverriden1, exclude="/*.jpg|", priority=300
+    "example.com", instead_of=POTopLevelOverriden1, exclude="/*.jpg|", priority=300
 )
 class POTopLevel1(POBase):
-    expected_overrides = POTopLevelOverriden1
+    expected_instead_of = POTopLevelOverriden1
     expected_patterns = Patterns(["example.com"], ["/*.jpg|"], priority=300)
+    expected_to_return = None
     expected_meta = {}  # type: ignore
 
 
-@handle_urls("example.com", overrides=POTopLevelOverriden2)
+@handle_urls("example.com", instead_of=POTopLevelOverriden2)
 class POTopLevel2(POBase):
-    expected_overrides = POTopLevelOverriden2
+    expected_instead_of = POTopLevelOverriden2
     expected_patterns = Patterns(["example.com"])
+    expected_to_return = None
     expected_meta = {}  # type: ignore

tests/po_lib/a_module.py

@@ -8,8 +8,9 @@ class POModuleOverriden(ItemPage):
     ...
 
 
-@handle_urls("example.com", overrides=POModuleOverriden, extra_arg="foo")
+@handle_urls("example.com", instead_of=POModuleOverriden, extra_arg="foo")
 class POModule(POBase):
-    expected_overrides = POModuleOverriden
+    expected_instead_of = POModuleOverriden
     expected_patterns = Patterns(["example.com"])
+    expected_to_return = None
     expected_meta = {"extra_arg": "foo"}  # type: ignore

tests/po_lib/nested_package/__init__.py

@@ -11,9 +11,10 @@ class PONestedPkgOverriden(ItemPage):
 @handle_urls(
     include=["example.com", "example.org"],
     exclude=["/*.jpg|"],
-    overrides=PONestedPkgOverriden,
+    instead_of=PONestedPkgOverriden,
 )
 class PONestedPkg(POBase):
-    expected_overrides = PONestedPkgOverriden
+    expected_instead_of = PONestedPkgOverriden
     expected_patterns = Patterns(["example.com", "example.org"], ["/*.jpg|"])
+    expected_to_return = None
     expected_meta = {}  # type: ignore

tests/po_lib/nested_package/a_nested_module.py

@@ -11,11 +11,12 @@ class PONestedModuleOverriden(ItemPage):
 @handle_urls(
     include=["example.com", "example.org"],
     exclude=["/*.jpg|"],
-    overrides=PONestedModuleOverriden,
+    instead_of=PONestedModuleOverriden,
 )
 class PONestedModule(POBase):
-    expected_overrides = PONestedModuleOverriden
+    expected_instead_of = PONestedModuleOverriden
     expected_patterns = Patterns(
         include=["example.com", "example.org"], exclude=["/*.jpg|"]
     )
+    expected_to_return = None
     expected_meta = {}  # type: ignore

tests/po_lib_sub/__init__.py

@@ -9,7 +9,7 @@
 
 
 class POBase(ItemPage):
-    expected_overrides: Type[ItemPage]
+    expected_instead_of: Type[ItemPage]
     expected_patterns: Patterns
     expected_meta: Dict[str, Any]
 
@@ -18,8 +18,9 @@ class POLibSubOverriden(ItemPage):
     ...
 
 
-@handle_urls("sub_example.com", overrides=POLibSubOverriden)
+@handle_urls("sub_example.com", instead_of=POLibSubOverriden)
 class POLibSub(POBase):
-    expected_overrides = POLibSubOverriden
+    expected_instead_of = POLibSubOverriden
     expected_patterns = Patterns(["sub_example.com"])
+    expected_to_return = None
     expected_meta = {}  # type: ignore

tests/po_lib_to_return/__init__.py

@@ -0,0 +1,160 @@
+import attrs
+from url_matcher import Patterns
+
+from web_poet import Injectable, ItemPage, Returns, field, handle_urls, item_from_fields
+
+
+@attrs.define
+class Product:
+    name: str
+    price: float
+
+
+@attrs.define
+class ProductSimilar:
+    name: str
+    price: float
+
+
+@attrs.define
+class ProductMoreFields(Product):
+    brand: str
+
+
+@attrs.define
+class ProductLessFields:
+    name: str
+
+
+@handle_urls("example.com")
+class ProductPage(ItemPage[Product]):
+    """A base PO to populate the Product item's fields."""
+
+    expected_instead_of = None
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = Product
+    expected_meta = {}
+
+    @field
+    def name(self) -> str:
+        return "name"
+
+    @field
+    def price(self) -> float:
+        return 12.99
+
+
+@handle_urls("example.com", instead_of=ProductPage)
+class ImprovedProductPage(ProductPage):
+    """A custom PO inheriting from a base PO which alters some field values."""
+
+    expected_instead_of = ProductPage
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = Product
+    expected_meta = {}
+
+    @field
+    def name(self) -> str:
+        return "improved name"
+
+
+@handle_urls("example.com", instead_of=ProductPage)
+class SimilarProductPage(ProductPage, Returns[ProductSimilar]):
+    """A custom PO inheriting from a base PO returning the same fields but in
+    a different item class.
+    """
+
+    expected_instead_of = ProductPage
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = ProductSimilar
+    expected_meta = {}
+
+
+@handle_urls("example.com", instead_of=ProductPage)
+class MoreProductPage(ProductPage, Returns[ProductMoreFields]):
+    """A custom PO inheriting from a base PO returning more items using a
+    different item class.
+    """
+
+    expected_instead_of = ProductPage
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = ProductMoreFields
+    expected_meta = {}
+
+    @field
+    def brand(self) -> str:
+        return "brand"
+
+
+@handle_urls("example.com", instead_of=ProductPage)
+class LessProductPage(
+    ProductPage, Returns[ProductLessFields], skip_nonitem_fields=True
+):
+    """A custom PO inheriting from a base PO returning less items using a
+    different item class.
+    """
+
+    expected_instead_of = ProductPage
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = ProductLessFields
+    expected_meta = {}
+
+    @field
+    def brand(self) -> str:
+        return "brand"
+
+
+@handle_urls("example.com", instead_of=ProductPage, to_return=ProductSimilar)
+class CustomProductPage(ProductPage, Returns[Product]):
+    """A custom PO inheriting from a base PO returning the same fields but in
+    a different item class.
+
+    This PO is the same with ``SimilarProductPage`` but passes a ``to_return``
+    in the ``@handle_urls`` decorator.
+
+    This tests the case that the type inside ``Returns`` should be followed and
+    the ``to_return`` parameter from ``@handle_urls`` is ignored.
+    """
+
+    expected_instead_of = ProductPage
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = Product
+    expected_meta = {}
+
+
+@handle_urls("example.com", instead_of=ProductPage, to_return=ProductSimilar)
+class CustomProductPageNoReturns(ProductPage):
+    """Same case as with ``CustomProductPage`` but doesn't inherit from
+    ``Returns[Product]``.
+    """
+
+    expected_instead_of = ProductPage
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = Product
+    expected_meta = {}
+
+
+@handle_urls("example.com", to_return=Product)
+class CustomProductPageDataTypeOnly(Injectable):
+    """A PO that doesn't inherit from ``ItemPage`` and ``WebPage`` which means
+    it doesn't inherit from the ``Returns`` class.
+
+    This tests the case that the ``to_return`` parameter in ``@handle_urls``
+    should properly use it in the rules.
+    """
+
+    expected_instead_of = None
+    expected_patterns = Patterns(["example.com"])
+    expected_to_return = Product
+    expected_meta = {}
+
+    @field
+    def name(self) -> str:
+        return "name"
+
+    @field
+    def price(self) -> float:
+        return 12.99
+
+    async def to_item(self) -> Product:
+        return await item_from_fields(self, item_cls=Product)

tests/test_fields.py

@@ -4,6 +4,20 @@
 import attrs
 import pytest
 
+from tests.po_lib_to_return import (
+    CustomProductPage,
+    CustomProductPageDataTypeOnly,
+    CustomProductPageNoReturns,
+    ImprovedProductPage,
+    LessProductPage,
+    MoreProductPage,
+    Product,
+    ProductLessFields,
+    ProductMoreFields,
+    ProductPage,
+    ProductSimilar,
+    SimilarProductPage,
+)
 from web_poet import (
     HttpResponse,
     Injectable,
@@ -368,3 +382,43 @@ def field_foo_cached(self):
     assert page.field_foo == "foo"
     assert page.field_foo_meta == "foo"
     assert page.field_foo_cached == "foo"
+
+
+@pytest.mark.asyncio
+async def test_field_with_handle_urls() -> None:
+
+    page = ProductPage()
+    assert page.name == "name"
+    assert page.price == 12.99
+    assert await page.to_item() == Product(name="name", price=12.99)
+
+    page = ImprovedProductPage()
+    assert page.name == "improved name"
+    assert page.price == 12.99
+    assert await page.to_item() == Product(name="improved name", price=12.99)
+
+    page = SimilarProductPage()
+    assert page.name == "name"
+    assert page.price == 12.99
+    assert await page.to_item() == ProductSimilar(name="name", price=12.99)
+
+    page = MoreProductPage()
+    assert page.name == "name"
+    assert page.price == 12.99
+    assert page.brand == "brand"
+    assert await page.to_item() == ProductMoreFields(
+        name="name", price=12.99, brand="brand"
+    )
+
+    page = LessProductPage()
+    assert page.name == "name"
+    assert await page.to_item() == ProductLessFields(name="name")
+
+    for page in [  # type: ignore[assignment]
+        CustomProductPage(),
+        CustomProductPageNoReturns(),
+        CustomProductPageDataTypeOnly(),
+    ]:
+        assert page.name == "name"
+        assert page.price == 12.99
+        assert await page.to_item() == Product(name="name", price=12.99)