diff --git a/.gitignore b/.gitignore index 4b56efc6da..c66d2bd194 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,22 @@ -.venv -.project -.spyderproject -.pydevproject -.vscode -.settings/ -.vscode/ -build/ -dist/ -doc/ *.egg-info *.mo *.pyc *~ -/cover -/.coverage -/tags -.tags* +.coverage +.mypy_cache .noseids +.project +.pydevproject .pytest_cache -.mypy_cache +.settings +.spyderproject +.tags* +.venv +.vscode +.vscode +build +cover +dist +doc performance.json +tags diff --git a/CHANGELOG.md b/CHANGELOG.md index 153dd51231..684c94323a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 35.6.0 [#1054](https://github.com/openfisca/openfisca-core/pull/1054) + +#### New Features + +- Introduce `openfisca_core.types` + +#### Documentation + +- Complete typing of the commons module + +#### Dependencies + +- `nptyping` + - To add backport-support for numpy typing + - Can be removed once lower-bound numpy version is 1.21+ + +- `typing_extensions` + - To add backport-support for `typing.Protocol` and `typing.Literal` + - Can be removed once lower-bound python version is 3.8+ + ### 35.5.5 [#1055](https://github.com/openfisca/openfisca-core/pull/1055) #### Documentation diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index 3a93477bc5..6a90622147 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,12 +1,22 @@ +from typing import Any, Dict, Sequence, TypeVar + import numpy +from openfisca_core.types import ArrayLike, ArrayType + +T = TypeVar("T") + -def apply_thresholds(input, thresholds, choices): +def apply_thresholds( + input: ArrayType[float], + thresholds: ArrayLike[float], + choices: ArrayLike[float], + ) -> ArrayType[float]: """Makes a choice based on an input and thresholds. - From a list of ``choices``, this function selects one of these values based on a list - of inputs, depending on the value of each ``input`` within a list of - ``thresholds``. + From a list of ``choices``, this function selects one of these values + based on a list of inputs, depending on the value of each ``input`` within + a list of ``thresholds``. Args: input: A list of inputs to make a choice from. @@ -30,16 +40,24 @@ def apply_thresholds(input, thresholds, choices): """ + condlist: Sequence[ArrayType[bool]] condlist = [input <= threshold for threshold in thresholds] + if len(condlist) == len(choices) - 1: - # If a choice is provided for input > highest threshold, last condition must be true to return it. + # If a choice is provided for input > highest threshold, last condition + # must be true to return it. condlist += [True] + assert len(condlist) == len(choices), \ - "apply_thresholds must be called with the same number of thresholds than choices, or one more choice" + " ".join([ + "'apply_thresholds' must be called with the same number of", + "thresholds than choices, or one more choice.", + ]) + return numpy.select(condlist, choices) -def concat(this, that): +def concat(this: ArrayLike[str], that: ArrayLike[str]) -> ArrayType[str]: """Concatenates the values of two arrays. Args: @@ -58,15 +76,23 @@ def concat(this, that): """ - if isinstance(this, numpy.ndarray) and not numpy.issubdtype(this.dtype, numpy.str): + if isinstance(this, numpy.ndarray) and \ + not numpy.issubdtype(this.dtype, numpy.str_): + this = this.astype('str') - if isinstance(that, numpy.ndarray) and not numpy.issubdtype(that.dtype, numpy.str): + + if isinstance(that, numpy.ndarray) and \ + not numpy.issubdtype(that.dtype, numpy.str_): + that = that.astype('str') - return numpy.core.defchararray.add(this, that) + return numpy.char.add(this, that) -def switch(conditions, value_by_condition): +def switch( + conditions: ArrayType[Any], + value_by_condition: Dict[float, T], + ) -> ArrayType[T]: """Mimicks a switch statement. Given an array of conditions, returns an array of the same size, @@ -77,7 +103,7 @@ def switch(conditions, value_by_condition): value_by_condition: Values to replace for each condition. Returns: - :obj:`numpy.ndarray` of :obj:`float`: + :obj:`numpy.ndarray`: An array with the replaced values. Raises: @@ -92,9 +118,11 @@ def switch(conditions, value_by_condition): """ assert len(value_by_condition) > 0, \ - "switch must be called with at least one value" + "'switch' must be called with at least one value." + condlist = [ conditions == condition for condition in value_by_condition.keys() ] + return numpy.select(condlist, value_by_condition.values()) diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index 4999e6cfe4..dd05cea11b 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,7 +1,11 @@ -import numpy +from typing import TypeVar +from openfisca_core.types import ArrayType -def empty_clone(original): +T = TypeVar("T") + + +def empty_clone(original: T) -> T: """Creates an empty instance of the same class of the original object. Args: @@ -25,18 +29,21 @@ def empty_clone(original): """ - class Dummy(original.__class__): - """Dummy class for empty cloning.""" + Dummy: object + new: T - def __init__(self) -> None: - ... + Dummy = type( + "Dummy", + (original.__class__,), + {"__init__": lambda self: None}, + ) new = Dummy() new.__class__ = original.__class__ return new -def stringify_array(array: numpy.ndarray) -> str: +def stringify_array(array: ArrayType) -> str: """Generates a clean string representation of a numpy array. Args: diff --git a/openfisca_core/commons/rates.py b/openfisca_core/commons/rates.py index bb94e24776..d682824207 100644 --- a/openfisca_core/commons/rates.py +++ b/openfisca_core/commons/rates.py @@ -1,7 +1,15 @@ +from typing import Optional + import numpy +from openfisca_core.types import ArrayLike, ArrayType + -def average_rate(target = None, varying = None, trim = None): +def average_rate( + target: ArrayType[float], + varying: ArrayLike[float], + trim: Optional[ArrayLike[float]] = None, + ) -> ArrayType[float]: """Computes the average rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -33,15 +41,32 @@ def average_rate(target = None, varying = None, trim = None): """ + average_rate: ArrayType[float] + average_rate = 1 - target / varying + if trim is not None: - average_rate = numpy.where(average_rate <= max(trim), average_rate, numpy.nan) - average_rate = numpy.where(average_rate >= min(trim), average_rate, numpy.nan) + + average_rate = numpy.where( + average_rate <= max(trim), + average_rate, + numpy.nan, + ) + + average_rate = numpy.where( + average_rate >= min(trim), + average_rate, + numpy.nan, + ) return average_rate -def marginal_rate(target = None, varying = None, trim = None): +def marginal_rate( + target: ArrayType[float], + varying: ArrayType[float], + trim: Optional[ArrayLike[float]] = None, + ) -> ArrayType[float]: """Computes the marginal rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -73,9 +98,26 @@ def marginal_rate(target = None, varying = None, trim = None): """ - marginal_rate = 1 - (target[:-1] - target[1:]) / (varying[:-1] - varying[1:]) + marginal_rate: ArrayType[float] + + marginal_rate = ( + + 1 + - (target[:-1] - target[1:]) + / (varying[:-1] - varying[1:]) + ) + if trim is not None: - marginal_rate = numpy.where(marginal_rate <= max(trim), marginal_rate, numpy.nan) - marginal_rate = numpy.where(marginal_rate >= min(trim), marginal_rate, numpy.nan) + + marginal_rate = numpy.where( + marginal_rate <= max(trim), + marginal_rate, + numpy.nan, + ) + + marginal_rate = numpy.where( + marginal_rate >= min(trim), + marginal_rate, + numpy.nan, + ) return marginal_rate diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py new file mode 100644 index 0000000000..e14cfea65d --- /dev/null +++ b/openfisca_core/types/__init__.py @@ -0,0 +1,45 @@ +"""Data types and protocols used by OpenFisca Core. + +The type definitions included in this sub-package are intented for +contributors, to help them better understand and document contracts +and expected behaviours. + +Official Public API: + * ``ArrayLike`` + * :attr:`.ArrayType` + +Note: + How imports are being used today:: + + from openfisca_core.types import * # Bad + from openfisca_core.types.data_types.arrays import ArrayLike # Bad + + + The previous examples provoke cyclic dependency problems, that prevents us + from modularizing the different components of the library, so as to make + them easier to test and to maintain. + + How could them be used after the next major release:: + + from openfisca_core.types import ArrayLike + + ArrayLike # Good: import types as publicly exposed + + .. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. + + .. _PEP8#Imports: + https://www.python.org/dev/peps/pep-0008/#imports + + .. _OpenFisca's Styleguide: + https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md + +""" + +# Official Public API + +from .data_types import ( # noqa: F401 + ArrayLike, + ArrayType, + ) + +__all__ = ["ArrayLike", "ArrayType"] diff --git a/openfisca_core/types/data_types/__init__.py b/openfisca_core/types/data_types/__init__.py new file mode 100644 index 0000000000..6dd38194e3 --- /dev/null +++ b/openfisca_core/types/data_types/__init__.py @@ -0,0 +1 @@ +from .arrays import ArrayLike, ArrayType # noqa: F401 diff --git a/openfisca_core/types/data_types/arrays.py b/openfisca_core/types/data_types/arrays.py new file mode 100644 index 0000000000..5cfef639c5 --- /dev/null +++ b/openfisca_core/types/data_types/arrays.py @@ -0,0 +1,51 @@ +from typing import Sequence, TypeVar, Union + +from nptyping import types, NDArray as ArrayType + +import numpy + +T = TypeVar("T", bool, bytes, float, int, object, str) + +types._ndarray_meta._Type = Union[type, numpy.dtype, TypeVar] + +ArrayLike = Union[ArrayType[T], Sequence[T]] +""":obj:`typing.Generic`: Type of any castable to :class:`numpy.ndarray`. + +These include any :obj:`numpy.ndarray` and sequences (like +:obj:`list`, :obj:`tuple`, and so on). + +Examples: + >>> ArrayLike[float] + typing.Union[numpy.ndarray, typing.Sequence[float]] + + >>> ArrayLike[str] + typing.Union[numpy.ndarray, typing.Sequence[str]] + +Note: + It is possible since numpy version 1.21 to specify the type of an + array, thanks to `numpy.typing.NDArray`_:: + + from numpy.typing import NDArray + NDArray[numpy.float64] + + `mypy`_ provides `duck type compatibility`_, so an :obj:`int` is + considered to be valid whenever a :obj:`float` is expected. + +Todo: + * Refactor once numpy version >= 1.21 is used. + +.. versionadded:: 35.5.0 + +.. versionchanged:: 35.6.0 + Moved to :mod:`.types` + +.. _mypy: + https://mypy.readthedocs.io/en/stable/ + +.. _duck type compatibility: + https://mypy.readthedocs.io/en/stable/duck_type_compatibility.html + +.. _numpy.typing.NDArray: + https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray + +""" diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 886072bb2b..115c6267bb 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -1,5 +1,5 @@ ## Lint the codebase. -lint: check-syntax-errors check-style lint-doc check-types +lint: check-syntax-errors check-style lint-doc check-types lint-typing-strict @$(call print_pass,$@:) ## Compile python files to check for syntax errors. @@ -39,6 +39,22 @@ check-types: @mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) +## Run static type checkers for type errors (strict). +lint-typing-strict: \ + lint-typing-strict-commons \ + lint-typing-strict-types \ + ; + +## Run static type checkers for type errors (strict). +lint-typing-strict-%: + @$(call print_help,$(subst $*,%,$@:)) + @mypy \ + --cache-dir .mypy_cache-openfisca_core.$* \ + --implicit-reexport \ + --strict \ + --package openfisca_core.$* + @$(call print_pass,$@:) + ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") @$(call print_help,$@:) diff --git a/setup.cfg b/setup.cfg index 213867581a..108d152f59 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,9 +31,11 @@ testpaths = openfisca_core/commons tests [mypy] ignore_missing_imports = True +install_types = True +non_interactive = True [mypy-openfisca_core.commons.tests.*] -ignore_errors = True +ignore_errors = True [mypy-openfisca_core.scripts.*] -ignore_errors = True +ignore_errors = True diff --git a/setup.py b/setup.py index acb886dde0..fb03815a1d 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,14 @@ general_requirements = [ 'dpath >= 1.5.0, < 2.0.0', + 'nptyping == 1.4.4', 'numexpr >= 2.7.0, <= 3.0', 'numpy >= 1.11, < 1.21', 'psutil >= 5.4.7, < 6.0.0', 'pytest >= 4.4.1, < 6.0.0', # For openfisca test 'PyYAML >= 3.10', 'sortedcontainers == 2.2.2', + 'typing-extensions == 3.10.0.2', ] api_requirements = [ @@ -29,8 +31,8 @@ 'flake8-bugbear >= 19.3.0, < 20.0.0', 'flake8-docstrings == 1.6.0', 'flake8-print >= 3.1.0, < 4.0.0', - 'flake8-rst-docstrings < 1.0.0', - 'mypy >= 0.701, < 0.800', + 'flake8-rst-docstrings == 0.2.3', + 'mypy == 0.910', 'openfisca-country-template >= 3.10.0, < 4.0.0', 'openfisca-extension-template >= 1.2.0rc0, < 2.0.0', 'pylint == 2.10.2', @@ -39,7 +41,7 @@ setup( name = 'OpenFisca-Core', - version = '35.5.5', + version = '35.6.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [