Merge pull request #1054 from openfisca/typing/commons

[1/17] Improve commons module typing

.gitignore

@@ -1,22 +1,22 @@
-.venv
-.project
-.spyderproject
-.pydevproject
-.vscode
-.settings/
-.vscode/
-build/
-dist/
-doc/
 *.egg-info
 *.mo
 *.pyc
 *~
-/cover
-/.coverage
-/tags
-.tags*
+.coverage
+.mypy_cache
 .noseids
+.project
+.pydevproject
 .pytest_cache
-.mypy_cache
+.settings
+.spyderproject
+.tags*
+.venv
+.vscode
+.vscode
+build
+cover
+dist
+doc
 performance.json
+tags

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 35.6.0 [#1054](https://github.com/openfisca/openfisca-core/pull/1054)
+
+#### New Features
+
+- Introduce `openfisca_core.types`
+
+#### Documentation
+
+- Complete typing of the commons module
+
+#### Dependencies
+
+- `nptyping`
+  - To add backport-support for numpy typing
+  - Can be removed once lower-bound numpy version is 1.21+
+
+- `typing_extensions`
+  - To add backport-support for `typing.Protocol` and `typing.Literal`
+  - Can be removed once lower-bound python version is 3.8+
+
 ### 35.5.5 [#1055](https://github.com/openfisca/openfisca-core/pull/1055)
 
 #### Documentation

openfisca_core/commons/formulas.py

@@ -1,12 +1,22 @@
+from typing import Any, Dict, Sequence, TypeVar
+
 import numpy
 
+from openfisca_core.types import ArrayLike, ArrayType
+
+T = TypeVar("T")
+
 
-def apply_thresholds(input, thresholds, choices):
+def apply_thresholds(
+        input: ArrayType[float],
+        thresholds: ArrayLike[float],
+        choices: ArrayLike[float],
+        ) -> ArrayType[float]:
     """Makes a choice based on an input and thresholds.
 
-    From a list of ``choices``, this function selects one of these values based on a list
-    of inputs, depending on the value of each ``input`` within a list of
-    ``thresholds``.
+    From a list of ``choices``, this function selects one of these values
+    based on a list of inputs, depending on the value of each ``input`` within
+    a list of ``thresholds``.
 
     Args:
         input: A list of inputs to make a choice from.
@@ -30,16 +40,24 @@ def apply_thresholds(input, thresholds, choices):
 
     """
 
+    condlist: Sequence[ArrayType[bool]]
     condlist = [input <= threshold for threshold in thresholds]
+
     if len(condlist) == len(choices) - 1:
-        # If a choice is provided for input > highest threshold, last condition must be true to return it.
+        # If a choice is provided for input > highest threshold, last condition
+        # must be true to return it.
         condlist += [True]
+
     assert len(condlist) == len(choices), \
-        "apply_thresholds must be called with the same number of thresholds than choices, or one more choice"
+        " ".join([
+            "'apply_thresholds' must be called with the same number of",
+            "thresholds than choices, or one more choice.",
+            ])
+
     return numpy.select(condlist, choices)
 
 
-def concat(this, that):
+def concat(this: ArrayLike[str], that: ArrayLike[str]) -> ArrayType[str]:
     """Concatenates the values of two arrays.
 
     Args:
@@ -58,15 +76,23 @@ def concat(this, that):
 
     """
 
-    if isinstance(this, numpy.ndarray) and not numpy.issubdtype(this.dtype, numpy.str):
+    if isinstance(this, numpy.ndarray) and \
+       not numpy.issubdtype(this.dtype, numpy.str_):
+
         this = this.astype('str')
-    if isinstance(that, numpy.ndarray) and not numpy.issubdtype(that.dtype, numpy.str):
+
+    if isinstance(that, numpy.ndarray) and \
+       not numpy.issubdtype(that.dtype, numpy.str_):
+
         that = that.astype('str')
 
-    return numpy.core.defchararray.add(this, that)
+    return numpy.char.add(this, that)
 
 
-def switch(conditions, value_by_condition):
+def switch(
+        conditions: ArrayType[Any],
+        value_by_condition: Dict[float, T],
+        ) -> ArrayType[T]:
     """Mimicks a switch statement.
 
     Given an array of conditions, returns an array of the same size,
@@ -77,7 +103,7 @@ def switch(conditions, value_by_condition):
         value_by_condition: Values to replace for each condition.
 
     Returns:
-        :obj:`numpy.ndarray` of :obj:`float`:
+        :obj:`numpy.ndarray`:
         An array with the replaced values.
 
     Raises:
@@ -92,9 +118,11 @@ def switch(conditions, value_by_condition):
     """
 
     assert len(value_by_condition) > 0, \
-        "switch must be called with at least one value"
+        "'switch' must be called with at least one value."
+
     condlist = [
         conditions == condition
         for condition in value_by_condition.keys()
         ]
+
     return numpy.select(condlist, value_by_condition.values())

openfisca_core/commons/misc.py

@@ -1,7 +1,11 @@
-import numpy
+from typing import TypeVar
 
+from openfisca_core.types import ArrayType
 
-def empty_clone(original):
+T = TypeVar("T")
+
+
+def empty_clone(original: T) -> T:
     """Creates an empty instance of the same class of the original object.
 
     Args:
@@ -25,18 +29,21 @@ def empty_clone(original):
 
     """
 
-    class Dummy(original.__class__):
-        """Dummy class for empty cloning."""
+    Dummy: object
+    new: T
 
-        def __init__(self) -> None:
-            ...
+    Dummy = type(
+        "Dummy",
+        (original.__class__,),
+        {"__init__": lambda self: None},
+        )
 
     new = Dummy()
     new.__class__ = original.__class__
     return new
 
 
-def stringify_array(array: numpy.ndarray) -> str:
+def stringify_array(array: ArrayType) -> str:
     """Generates a clean string representation of a numpy array.
 
     Args:

openfisca_core/commons/rates.py

@@ -1,7 +1,15 @@
+from typing import Optional
+
 import numpy
 
+from openfisca_core.types import ArrayLike, ArrayType
+
 
-def average_rate(target = None, varying = None, trim = None):
+def average_rate(
+        target: ArrayType[float],
+        varying: ArrayLike[float],
+        trim: Optional[ArrayLike[float]] = None,
+        ) -> ArrayType[float]:
     """Computes the average rate of a target net income.
 
     Given a ``target`` net income, and according to the ``varying`` gross
@@ -33,15 +41,32 @@ def average_rate(target = None, varying = None, trim = None):
 
     """
 
+    average_rate: ArrayType[float]
+
     average_rate = 1 - target / varying
+
     if trim is not None:
-        average_rate = numpy.where(average_rate <= max(trim), average_rate, numpy.nan)
-        average_rate = numpy.where(average_rate >= min(trim), average_rate, numpy.nan)
+
+        average_rate = numpy.where(
+            average_rate <= max(trim),
+            average_rate,
+            numpy.nan,
+            )
+
+        average_rate = numpy.where(
+            average_rate >= min(trim),
+            average_rate,
+            numpy.nan,
+            )
 
     return average_rate
 
 
-def marginal_rate(target = None, varying = None, trim = None):
+def marginal_rate(
+        target: ArrayType[float],
+        varying: ArrayType[float],
+        trim: Optional[ArrayLike[float]] = None,
+        ) -> ArrayType[float]:
     """Computes the marginal rate of a target net income.
 
     Given a ``target`` net income, and according to the ``varying`` gross
@@ -73,9 +98,26 @@ def marginal_rate(target = None, varying = None, trim = None):
 
     """
 
-    marginal_rate = 1 - (target[:-1] - target[1:]) / (varying[:-1] - varying[1:])
+    marginal_rate: ArrayType[float]
+
+    marginal_rate = (
+        + 1
+        - (target[:-1] - target[1:])
+        / (varying[:-1] - varying[1:])
+        )
+
     if trim is not None:
-        marginal_rate = numpy.where(marginal_rate <= max(trim), marginal_rate, numpy.nan)
-        marginal_rate = numpy.where(marginal_rate >= min(trim), marginal_rate, numpy.nan)
+
+        marginal_rate = numpy.where(
+            marginal_rate <= max(trim),
+            marginal_rate,
+            numpy.nan,
+            )
+
+        marginal_rate = numpy.where(
+            marginal_rate >= min(trim),
+            marginal_rate,
+            numpy.nan,
+            )
 
     return marginal_rate

openfisca_core/types/__init__.py

@@ -0,0 +1,45 @@
+"""Data types and protocols used by OpenFisca Core.
+
+The type definitions included in this sub-package are intented for
+contributors, to help them better understand and document contracts
+and expected behaviours.
+
+Official Public API:
+    * ``ArrayLike``
+    * :attr:`.ArrayType`
+
+Note:
+    How imports are being used today::
+
+        from openfisca_core.types import *  # Bad
+        from openfisca_core.types.data_types.arrays import ArrayLike  # Bad
+
+
+    The previous examples provoke cyclic dependency problems, that prevents us
+    from modularizing the different components of the library, so as to make
+    them easier to test and to maintain.
+
+    How could them be used after the next major release::
+
+        from openfisca_core.types import ArrayLike
+
+        ArrayLike # Good: import types as publicly exposed
+
+    .. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_.
+
+    .. _PEP8#Imports:
+        https://www.python.org/dev/peps/pep-0008/#imports
+
+    .. _OpenFisca's Styleguide:
+        https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md
+
+"""
+
+# Official Public API
+
+from .data_types import (  # noqa: F401
+    ArrayLike,
+    ArrayType,
+    )
+
+__all__ = ["ArrayLike", "ArrayType"]

openfisca_core/types/data_types/__init__.py

@@ -0,0 +1 @@
+from .arrays import ArrayLike, ArrayType  # noqa: F401

openfisca_core/types/data_types/arrays.py

@@ -0,0 +1,51 @@
+from typing import Sequence, TypeVar, Union
+
+from nptyping import types, NDArray as ArrayType
+
+import numpy
+
+T = TypeVar("T", bool, bytes, float, int, object, str)
+
+types._ndarray_meta._Type = Union[type, numpy.dtype, TypeVar]
+
+ArrayLike = Union[ArrayType[T], Sequence[T]]
+""":obj:`typing.Generic`: Type of any castable to :class:`numpy.ndarray`.
+
+These include any :obj:`numpy.ndarray` and sequences (like
+:obj:`list`, :obj:`tuple`, and so on).
+
+Examples:
+    >>> ArrayLike[float]
+    typing.Union[numpy.ndarray, typing.Sequence[float]]
+
+    >>> ArrayLike[str]
+    typing.Union[numpy.ndarray, typing.Sequence[str]]
+
+Note:
+    It is possible since numpy version 1.21 to specify the type of an
+    array, thanks to `numpy.typing.NDArray`_::
+
+        from numpy.typing import NDArray
+        NDArray[numpy.float64]
+
+    `mypy`_ provides `duck type compatibility`_, so an :obj:`int` is
+    considered to be valid whenever a :obj:`float` is expected.
+
+Todo:
+    * Refactor once numpy version >= 1.21 is used.
+
+.. versionadded:: 35.5.0
+
+.. versionchanged:: 35.6.0
+    Moved to :mod:`.types`
+
+.. _mypy:
+    https://mypy.readthedocs.io/en/stable/
+
+.. _duck type compatibility:
+    https://mypy.readthedocs.io/en/stable/duck_type_compatibility.html
+
+.. _numpy.typing.NDArray:
+    https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray
+
+"""