From 085487d1540201566a071ed3874246c00e064ef7 Mon Sep 17 00:00:00 2001 From: Ramon Hagenaars <37958579+ramonhagenaars@users.noreply.github.com> Date: Mon, 20 Feb 2023 22:56:17 +0100 Subject: [PATCH] Release/2.5.0 (#101) * Made the help text for functions with nptyping type hints become more helpful. * Updated some constraints. * Made the wheel test run only on 1 Python version (no differences expected between versions). * Updated tasks. * Allowed generic wildcard in structure expressions to allow "any columns". * Updated year in copyright statement * Attempt at enabling mypy-dataframe-tests on 3.11 on Actions. * Patch for invoke for python3.11 * Added note. * Added python3.12 to Actions * Woops * Update readme * Updated docs with global wildcard * Updated the packaging and distribution. * Version bump. * Documentation * Updated Readme. * Updated History. --------- Co-authored-by: Ramon --- .github/workflows/pythonapp.yml | 3 +- HISTORY.md | 7 +++ MANIFEST.in | 12 ++++ README.md | 16 ++--- USERDOCS.md | 53 ++++++++++++++-- constraints-3.10.txt | 6 +- constraints-3.11.txt | 6 +- constraints-3.7.txt | 4 +- constraints-3.8.txt | 6 +- constraints-3.9.txt | 6 +- nptyping/__init__.py | 2 +- nptyping/assert_isinstance.py | 2 +- nptyping/base_meta_classes.py | 9 ++- nptyping/error.py | 2 +- nptyping/ndarray.py | 7 ++- nptyping/ndarray.pyi | 2 +- nptyping/nptyping_type.py | 2 +- nptyping/package_info.py | 4 +- nptyping/pandas_/dataframe.py | 7 ++- nptyping/pandas_/dataframe.pyi | 2 +- nptyping/pandas_/typing_.py | 2 +- nptyping/recarray.py | 7 ++- nptyping/recarray.pyi | 2 +- nptyping/shape.py | 2 +- nptyping/shape.pyi | 2 +- nptyping/shape_expression.py | 2 +- nptyping/structure.py | 16 ++++- nptyping/structure.pyi | 2 +- nptyping/structure_expression.py | 93 ++++++++++++++++++++-------- nptyping/typing_.py | 2 +- nptyping/typing_.pyi | 2 +- setup.py | 10 +-- tasks.py | 20 ++++-- tests/pandas_/test_mypy_dataframe.py | 12 +--- tests/test_help_texts.py | 42 +++++++++++++ tests/test_structure_expression.py | 16 +++++ tests/test_wheel.py | 7 ++- 37 files changed, 298 insertions(+), 99 deletions(-) create mode 100644 MANIFEST.in create mode 100644 tests/test_help_texts.py diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index b31efea..f3a419f 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -8,8 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [ '3.7', '3.8', '3.9', '3.10' ] - # FIXME: 3.11 fails because of invoke. See https://github.com/pyinvoke/invoke/pull/877 + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] os: [ ubuntu-latest, macOS-latest, windows-latest ] name: Python ${{ matrix.python-version }} on ${{ matrix.os }} steps: diff --git a/HISTORY.md b/HISTORY.md index 8f3feff..9eccc6a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,12 @@ # History +## 2.5.0 (2023-02-20) + +- Added the column wildcard in structure expressions to allow expressing 'a structure with at least ...'. +- Fixed the `help` text for functions that use `nptyping` types as hints. +- Fixed the distribution of `dataframe.pyi` that was missing. +- Fixed the sdist to include tests and dependencies. + ## 2.4.1 (2022-11-16) - Fixed compatibility with `mypy==0.991`. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..4b89a16 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,12 @@ +# See also: +# https://packaging.python.org/guides/using-manifest-in/#how-files-are-included-in-an-sdist +include CONTRIBUTING.md +include HISTORY.md +include USERDOCS.md +include dependencies/* +include dependencies/**/* +include resources/* +include resources/**/* +include tests/*.py +include tests/**/*.py +recursive-exclude *.pyc diff --git a/README.md b/README.md index dddeca7..d59554c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@

-💡 *Type hints for `NumPy`*
-💡 *Type hints for `pandas.DataFrame`*
+🧊 *Type hints for `NumPy`*
+🐼 *Type hints for `pandas.DataFrame`*
💡 *Extensive dynamic type checks for dtypes shapes and structures*
+🚀 *[Jump to the Quickstart](https://github.com/ramonhagenaars/nptyping/blob/master/USERDOCS.md#Quickstart)* Example of a hinted `numpy.ndarray`: @@ -32,15 +33,14 @@ Example of a hinted `pandas.DataFrame`: >>> df: DataFrame[S["name: Str, x: Float, y: Float"]] ``` -⚠️`pandas.DataFrame` is not yet supported on Python 3.11. ### Installation -| Command | Description | -|:---------------------------------|-----------------------------------------------------------| -| `pip install nptyping` | Install the basics | -| `pip install nptyping[pandas]` | Install with pandas extension (⚠️Python 3.10 or lower) | -| `pip install nptyping[complete]` | Install with all extensions | +| Command | Description | +|:---------------------------------|-------------------------------| +| `pip install nptyping` | Install the basics | +| `pip install nptyping[pandas]` | Install with pandas extension | +| `pip install nptyping[complete]` | Install with all extensions | ### Instance checking diff --git a/USERDOCS.md b/USERDOCS.md index 16662cf..1f20f44 100644 --- a/USERDOCS.md +++ b/USERDOCS.md @@ -5,6 +5,7 @@ # *User documentation* * [Introduction](#Introduction) +* [Quickstart](#Quickstart) * [Usage](#Usage) * [NDArray](#NDArray) * [Shape expressions](#Shape-expressions) @@ -12,7 +13,7 @@ * [Validation](#Validation) * [Normalization](#Normalization) * [Variables](#Variables) - * [Wildcards](#Wildcards) + * [Wildcards](#Shape-Wildcards) * [N dimensions](#N-dimensions) * [Dimension breakdowns](#Dimension-breakdowns) * [Labels](#Labels) @@ -20,6 +21,7 @@ * [Structure expressions](#Structure-expressions) * [Syntax](#Syntax-structure-expressions) * [Subarrays](#Subarrays) + * [Wildcards](#Structure-Wildcards) * [RecArray](#RecArray) * [Pandas DataFrame](#Pandas-DataFrame) * [Examples](#Examples) @@ -43,6 +45,32 @@ raise your question [in a new issue](https://github.com/ramonhagenaars/nptyping/ You will find a lot of code blocks in this document. If you wonder why they are written the way they are (e.g. with the `>>>` and the `...`): all code blocks are tested using [doctest](https://docs.python.org/3/library/doctest.html). +## Quickstart +Install `nptyping` for the type hints and the recommended `beartype` for dynamic type checks: +```shell +pip install nptyping[complete], beartype +``` + +Use the combination of these packages to add type safety and readability: +```python +# File: myfile.py + +>>> from nptyping import DataFrame, Structure as S +>>> from beartype import beartype + +>>> @beartype # The function signature is now type safe +... def fun(df: DataFrame[S["a: int, b: str"]]) -> DataFrame[S["a: int, b: str"]]: +... return df + +``` + +On your production environments, run Python in optimized mode. This disables the type checks done by beartype and any +overhead it may cause: +```shell +python -OO myfile.py +``` +You're now good to go. You can sleep tight knowing that today you made your codebase safer and more transparent. + ## Usage ### NDArray @@ -173,7 +201,7 @@ They are interpreted from left to right. This means that in the last example, up A variable is a word that may contain underscores and digits as long as *it starts with an uppercase letter*. -#### Wildcards +#### Shape Wildcards A wildcard accepts any dimension size. It is denoted by the asterisk (`*`). Example: ```python >>> isinstance(random.randn(42, 43), NDArray[Shape["*, *"], Any]) @@ -331,7 +359,7 @@ NDArray[Any, Floating] ### Structure expressions You can denote the structure of a structured array using what we call a **structure expression**. This expression -- again a string - can be put into `Structure` and can then be used in an `NDArray`. +(again a string) can be put into `Structure` and can then be used in an `NDArray`. ```python >>> from nptyping import Structure @@ -389,7 +417,7 @@ True The syntax of a structure expression can be formalized in BNF. Extra whitespacing is allowed (e.g. around commas and colons), but this is not included in the schema below. ``` -structure-expression = +structure-expression = |"," fields = |"," field = ":"|"[""]:" combined-field-names = ","|"," @@ -420,6 +448,23 @@ True ``` +#### Structure Wildcards +You can use wildcards for field types or globally (for complete fields). +Here is an example of a wildcard for a field type: +```python +>>> Structure["anyType: *"] +Structure['anyType: *'] + +``` + +And here is an example with a global wildcard: +```python +>>> Structure["someType: int, *"] +Structure['someType: int, *'] + +``` +This expresses a structure that has *at least* a field `someType: int`. Any other fields are also accepted. + ### RecArray The `RecArray` corresponds to [numpy.recarray](https://numpy.org/doc/stable/reference/generated/numpy.recarray.html). It is an extension of `NDArray` and behaves similarly. A key difference is that with `RecArray`, the `Structure` OR diff --git a/constraints-3.10.txt b/constraints-3.10.txt index fa2403a..b797ac2 100644 --- a/constraints-3.10.txt +++ b/constraints-3.10.txt @@ -49,9 +49,9 @@ lazy-object-proxy==1.7.1 # via astroid mccabe==0.7.0 # via pylint -mypy==0.991 +mypy==1.0.0 # via -r ./dependencies\qa-requirements.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via # black # mypy @@ -81,7 +81,7 @@ pylint==2.15.2 # via -r ./dependencies\qa-requirements.txt pyparsing==3.0.9 # via packaging -pyright==1.1.273 +pyright==1.1.294 # via -r ./dependencies\qa-requirements.txt python-dateutil==2.8.2 # via pandas diff --git a/constraints-3.11.txt b/constraints-3.11.txt index 0313e02..88c5ac3 100644 --- a/constraints-3.11.txt +++ b/constraints-3.11.txt @@ -49,9 +49,9 @@ lazy-object-proxy==1.8.0 # via astroid mccabe==0.7.0 # via pylint -mypy==0.991 +mypy==1.0.0 # via -r ./dependencies\qa-requirements.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via # black # mypy @@ -81,7 +81,7 @@ pylint==2.15.5 # via -r ./dependencies\qa-requirements.txt pyparsing==3.0.9 # via packaging -pyright==1.1.279 +pyright==1.1.294 # via -r ./dependencies\qa-requirements.txt python-dateutil==2.8.2 # via pandas diff --git a/constraints-3.7.txt b/constraints-3.7.txt index d74f77d..848df45 100644 --- a/constraints-3.7.txt +++ b/constraints-3.7.txt @@ -56,7 +56,7 @@ mccabe==0.7.0 # via pylint mypy==0.991 # via -r ./dependencies\qa-requirements.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via # black # mypy @@ -86,7 +86,7 @@ pylint==2.15.2 # via -r ./dependencies\qa-requirements.txt pyparsing==3.0.9 # via packaging -pyright==1.1.273 +pyright==1.1.294 # via -r ./dependencies\qa-requirements.txt python-dateutil==2.8.2 # via pandas diff --git a/constraints-3.8.txt b/constraints-3.8.txt index 11d035d..11214f0 100644 --- a/constraints-3.8.txt +++ b/constraints-3.8.txt @@ -49,9 +49,9 @@ lazy-object-proxy==1.7.1 # via astroid mccabe==0.7.0 # via pylint -mypy==0.991 +mypy==1.0.0 # via -r ./dependencies\qa-requirements.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via # black # mypy @@ -81,7 +81,7 @@ pylint==2.15.2 # via -r ./dependencies\qa-requirements.txt pyparsing==3.0.9 # via packaging -pyright==1.1.273 +pyright==1.1.294 # via -r ./dependencies\qa-requirements.txt python-dateutil==2.8.2 # via pandas diff --git a/constraints-3.9.txt b/constraints-3.9.txt index 5d4d833..8ff20f7 100644 --- a/constraints-3.9.txt +++ b/constraints-3.9.txt @@ -49,9 +49,9 @@ lazy-object-proxy==1.7.1 # via astroid mccabe==0.7.0 # via pylint -mypy==0.991 +mypy==1.0.0 # via -r ./dependencies\qa-requirements.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via # black # mypy @@ -81,7 +81,7 @@ pylint==2.15.2 # via -r ./dependencies\qa-requirements.txt pyparsing==3.0.9 # via packaging -pyright==1.1.273 +pyright==1.1.294 # via -r ./dependencies\qa-requirements.txt python-dateutil==2.8.2 # via pandas diff --git a/nptyping/__init__.py b/nptyping/__init__.py index bbcc7ff..5fd5b2c 100644 --- a/nptyping/__init__.py +++ b/nptyping/__init__.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/assert_isinstance.py b/nptyping/assert_isinstance.py index 328255a..5ae4eb1 100644 --- a/nptyping/assert_isinstance.py +++ b/nptyping/assert_isinstance.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/base_meta_classes.py b/nptyping/base_meta_classes.py index a769673..b9ca13a 100644 --- a/nptyping/base_meta_classes.py +++ b/nptyping/base_meta_classes.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,9 +22,11 @@ SOFTWARE. """ from abc import ABCMeta, abstractmethod +from inspect import FrameInfo from typing import ( Any, Dict, + List, Optional, Set, Tuple, @@ -123,6 +125,11 @@ class SubscriptableMeta(ABCMeta): def _get_item(cls, item: Any) -> Tuple[Any, ...]: ... # pragma: no cover + def _get_module(cls, stack: List[FrameInfo], module: str) -> str: + # The magic below makes Python's help function display a meaningful + # text with nptyping types. + return "typing" if stack[1][3] == "formatannotation" else module + def _get_additional_values( cls, item: Any # pylint: disable=unused-argument ) -> Dict[str, Any]: diff --git a/nptyping/error.py b/nptyping/error.py index 446328b..237864e 100644 --- a/nptyping/error.py +++ b/nptyping/error.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/ndarray.py b/nptyping/ndarray.py index 95b371e..adeea12 100644 --- a/nptyping/ndarray.py +++ b/nptyping/ndarray.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -21,6 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import inspect from abc import ABC from typing import Any, Tuple @@ -64,6 +65,10 @@ class NDArrayMeta( __args__: Tuple[Shape, DType] _parameterized: bool + @property + def __module__(cls) -> str: + return cls._get_module(inspect.stack(), "nptyping.ndarray") + def _get_item(cls, item: Any) -> Tuple[Any, ...]: cls._check_item(item) shape, dtype = cls._get_from_tuple(item) diff --git a/nptyping/ndarray.pyi b/nptyping/ndarray.pyi index 2823d25..8d4e761 100644 --- a/nptyping/ndarray.pyi +++ b/nptyping/ndarray.pyi @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/nptyping_type.py b/nptyping/nptyping_type.py index 5acc754..35d4897 100644 --- a/nptyping/nptyping_type.py +++ b/nptyping/nptyping_type.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/package_info.py b/nptyping/package_info.py index fac3248..ee15ac8 100644 --- a/nptyping/package_info.py +++ b/nptyping/package_info.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ SOFTWARE. """ __title__ = "nptyping" -__version__ = "2.4.1" +__version__ = "2.5.0" __author__ = "Ramon Hagenaars" __author_email__ = "ramon.hagenaars@gmail.com" __description__ = "Type hints for NumPy." diff --git a/nptyping/pandas_/dataframe.py b/nptyping/pandas_/dataframe.py index dd6b634..e108b27 100644 --- a/nptyping/pandas_/dataframe.py +++ b/nptyping/pandas_/dataframe.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -21,6 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import inspect from abc import ABC from typing import Any, Tuple @@ -102,6 +103,10 @@ def __repr__(cls) -> str: structure_str = "Any" if structure is Any else structure return f"{cls.__name__}[{structure_str}]" + @property + def __module__(cls) -> str: + return cls._get_module(inspect.stack(), "nptyping.pandas_.dataframe") + def _check_item(cls, item: Any) -> None: # Check if the item is what we expect and raise if it is not. if not hasattr(item, "__args__"): diff --git a/nptyping/pandas_/dataframe.pyi b/nptyping/pandas_/dataframe.pyi index add9e4d..edab03f 100644 --- a/nptyping/pandas_/dataframe.pyi +++ b/nptyping/pandas_/dataframe.pyi @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/pandas_/typing_.py b/nptyping/pandas_/typing_.py index 21190fe..29f2188 100644 --- a/nptyping/pandas_/typing_.py +++ b/nptyping/pandas_/typing_.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/recarray.py b/nptyping/recarray.py index 699e662..b3bbbc5 100644 --- a/nptyping/recarray.py +++ b/nptyping/recarray.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -21,6 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import inspect from typing import Any, Tuple import numpy as np @@ -49,6 +50,10 @@ def _get_dtype(cls, dtype_candidate: Any) -> DType: ) return dtype_candidate + @property + def __module__(cls) -> str: + return cls._get_module(inspect.stack(), "nptyping.recarray") + def __instancecheck__( # pylint: disable=bad-mcs-method-argument self, instance: Any ) -> bool: diff --git a/nptyping/recarray.pyi b/nptyping/recarray.pyi index 7024443..4e67f8a 100644 --- a/nptyping/recarray.pyi +++ b/nptyping/recarray.pyi @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/shape.py b/nptyping/shape.py index c6281ea..5dc05f5 100644 --- a/nptyping/shape.py +++ b/nptyping/shape.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/shape.pyi b/nptyping/shape.pyi index e4a04e6..96d2eb6 100644 --- a/nptyping/shape.pyi +++ b/nptyping/shape.pyi @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/shape_expression.py b/nptyping/shape_expression.py index 148a355..bcaf79b 100644 --- a/nptyping/shape_expression.py +++ b/nptyping/shape_expression.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/structure.py b/nptyping/structure.py index 032d7f4..0b95dfb 100644 --- a/nptyping/structure.py +++ b/nptyping/structure.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -51,7 +51,10 @@ def _normalize_expression(cls, item: str) -> str: return normalize_structure_expression(item) def _get_additional_values(cls, item: Any) -> Dict[str, Any]: - return {"_type_per_name": create_name_to_type_dict(item)} + return { + "_type_per_name": create_name_to_type_dict(item), + "_has_wildcard": item.replace(" ", "").endswith(",*"), + } class Structure(NPTypingType, ABC, metaclass=StructureMeta): @@ -67,6 +70,15 @@ class Structure(NPTypingType, ABC, metaclass=StructureMeta): """ _type_per_name = {} + _has_wildcard = False + + @classmethod + def has_wildcard(cls) -> bool: + """ + Returns whether this Structure has a wildcard for any other columns. + :return: True if this Structure expresses "any other columns". + """ + return cls._has_wildcard @classmethod def get_types(cls) -> List[str]: diff --git a/nptyping/structure.pyi b/nptyping/structure.pyi index 4848cd2..415cf4e 100644 --- a/nptyping/structure.pyi +++ b/nptyping/structure.pyi @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/structure_expression.py b/nptyping/structure_expression.py index df12904..ee1236c 100644 --- a/nptyping/structure_expression.py +++ b/nptyping/structure_expression.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,8 +28,10 @@ TYPE_CHECKING, Any, Dict, + Generator, List, Mapping, + Tuple, Type, Union, ) @@ -88,33 +90,64 @@ def check_structure( # Add the wildcard to the lexicon. We want to do this here to keep # knowledge on wildcards in one place (this module). - type_per_name_with_wildcard = {**type_per_name, "*": object} # type: ignore[arg-type] - if set(target.get_names()) != set(fields.keys()): - return False - for name, dtype_tuple in fields.items(): - dtype = dtype_tuple[0] - target_type_name = target.get_type(name) - target_type_shape_match = re.search(_REGEX_FIELD_SHAPE, target_type_name) - actual_type = dtype.type - if target_type_shape_match: - if not dtype.subdtype: - # the dtype does not contain a shape. - return False - actual_type = dtype.subdtype[0].type - target_type_shape = target_type_shape_match.group(1) - shape_corresponds = check_shape(dtype.shape, Shape[target_type_shape]) - if not shape_corresponds: - return False - target_type_name = target_type_name.replace( - target_type_shape_match.group(0), "" - ) - check_type_name(target_type_name, type_per_name_with_wildcard) - target_type = type_per_name_with_wildcard[target_type_name] - if not issubclass(actual_type, target_type): + type_per_name_with_wildcard: Dict[str, type] = { + **type_per_name, + "*": object, + } # type: ignore[arg-type] + + if target.has_wildcard(): + # Check from the Target's perspective. All fields in the Target should be + # in the subject. + def iterator() -> Generator[Tuple[str, Tuple[np.dtype, int]], None, None]: # type: ignore[type-arg] # pylint: disable=line-too-long + for name_ in target.get_names(): + yield name_, fields.get(name_) # type: ignore[misc] + + else: + # Check from the subject's perspective. All fields in the subject + # should be in the target. + if set(target.get_names()) != set(fields.keys()): + return False + + def iterator() -> Generator[Tuple[str, Tuple[np.dtype, int]], None, None]: # type: ignore[type-arg] # pylint: disable=line-too-long + for name_, dtype_tuple_ in fields.items(): + yield name_, dtype_tuple_ # type: ignore[misc] + + for name, dtype_tuple in iterator(): + field_in_target_not_in_subject = dtype_tuple is None + if field_in_target_not_in_subject or not _check_structure_field( + name, dtype_tuple, target, type_per_name_with_wildcard + ): return False return True +def _check_structure_field( + name: str, + dtype_tuple: Tuple[np.dtype, int], # type: ignore[type-arg] + target: "Structure", + type_per_name_with_wildcard: Dict[str, type], +) -> bool: + dtype = dtype_tuple[0] + target_type_name = target.get_type(name) + target_type_shape_match = re.search(_REGEX_FIELD_SHAPE, target_type_name) + actual_type = dtype.type + if target_type_shape_match: + if not dtype.subdtype: + # the dtype does not contain a shape. + return False + actual_type = dtype.subdtype[0].type + target_type_shape = target_type_shape_match.group(1) + shape_corresponds = check_shape(dtype.shape, Shape[target_type_shape]) + if not shape_corresponds: + return False + target_type_name = target_type_name.replace( + target_type_shape_match.group(0), "" + ) + check_type_name(target_type_name, type_per_name_with_wildcard) + target_type = type_per_name_with_wildcard[target_type_name] + return issubclass(actual_type, target_type) + + def check_type_names( structure: "Structure", type_per_name: Dict[str, Type[object]] ) -> None: @@ -167,7 +200,11 @@ def normalize_structure_expression( structure_expression = re.sub(r"\s*", "", structure_expression) type_to_names_dict = _create_type_to_names_dict(structure_expression) normalized_structure_expression = _type_to_names_dict_to_str(type_to_names_dict) - return normalized_structure_expression.replace(",", ", ").replace(" ", " ") + result = normalized_structure_expression.replace(",", ", ").replace(" ", " ") + has_wildcard_end = structure_expression.replace(" ", "").endswith(",*") + if has_wildcard_end: + result += ", *" + return result def create_name_to_type_dict( @@ -295,4 +332,8 @@ def _type_to_names_dict_to_str(type_to_names_dict: Dict[str, List[str]]) -> str: _REGEX_FIELD = ( rf"(\s*{_REGEX_FIELD_LEFT}{_REGEX_FIELD_TYPE_POINTER}{_REGEX_FIELD_RIGHT}\s*)" ) -_REGEX_STRUCTURE_EXPRESSION = rf"^({_REGEX_FIELD}({_REGEX_SEPARATOR}{_REGEX_FIELD})*)$" +_REGEX_STRUCTURE_EXPRESSION = ( + rf"^({_REGEX_FIELD}" + rf"({_REGEX_SEPARATOR}{_REGEX_FIELD})*" + rf"({_REGEX_SEPARATOR}{_REGEX_FIELD_TYPE_WILDCARD})?)$" +) diff --git a/nptyping/typing_.py b/nptyping/typing_.py index fc7ce07..2639e07 100644 --- a/nptyping/typing_.py +++ b/nptyping/typing_.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nptyping/typing_.pyi b/nptyping/typing_.pyi index ffa5f98..fcf83ce 100644 --- a/nptyping/typing_.pyi +++ b/nptyping/typing_.pyi @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2022 Ramon Hagenaars +Copyright (c) 2023 Ramon Hagenaars Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/setup.py b/setup.py index c30d436..1c4cef7 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ import sys -from glob import glob from pathlib import Path from setuptools import find_packages, setup @@ -49,9 +48,6 @@ def _get_dependencies(dependency_file): # Dev: all extras for developers, including build and qa dependencies. extras["dev"] = [req for key, reqs in extras.items() for req in reqs] -pyi_files = [ - Path(pyi_file).name for pyi_file in glob("nptyping/**/*.pyi", recursive=True) -] setup( name=package_info["__title__"], @@ -64,11 +60,9 @@ def _get_dependencies(dependency_file): long_description_content_type="text/markdown", license=package_info["__license__"], package_data={ - "nptyping": pyi_files + ["py.typed"], + "": ["*.pyi", "py.typed"], }, - packages=find_packages( - exclude=("tests", "tests.*", "test_resources", "test_resources.*") - ), + packages=find_packages(include=("nptyping", "nptyping.*")), install_requires=_get_dependencies("requirements.txt"), extras_require=extras, python_requires=f">={supp_versions[0]}", diff --git a/tasks.py b/tasks.py index 63ddf0a..e29b787 100644 --- a/tasks.py +++ b/tasks.py @@ -28,6 +28,7 @@ from glob import glob from pathlib import Path +import invoke.tasks as invoke_tasks from invoke import task _ROOT = "nptyping" @@ -36,6 +37,12 @@ ) _DEFAULT_VENV = f".venv{_PY_VERSION_STR}" +if sys.version_info.minor >= 11: + # Patch invoke to replace a deprecated inspect function. + # FIXME: https://github.com/pyinvoke/invoke/pull/877 + invoke_tasks.inspect.getargspec = invoke_tasks.inspect.getfullargspec + + if os.name == "nt": _PY_SUFFIX = "\\Scripts\\python.exe" _PIP_SUFFIX = "\\Scripts\\pip.exe" @@ -111,6 +118,7 @@ def clean(context, py=None): shutil.rmtree("build", ignore_errors=True) shutil.rmtree(".mypy_cache", ignore_errors=True) shutil.rmtree(".pytest_cache", ignore_errors=True) + shutil.rmtree("__pycache__", ignore_errors=True) @task @@ -160,7 +168,7 @@ def wheel(context, py=None): """Build a wheel.""" print(f"Installing dependencies into: {_DEFAULT_VENV}") context.run(f"{get_py(py)} setup.py sdist") - context.run(f"{get_py(py)} setup.py bdist_wheel") + context.run(f"{get_pip(py)} wheel . --wheel-dir dist --no-deps") # QA TOOLS @@ -175,7 +183,7 @@ def test(context, py=None): @task -def doctest(context, py=None): +def doctest(context, py=None, verbose=False): """Run the doctests.""" # Check the README. context.run(f"{get_py(py)} -m doctest README.md") @@ -183,6 +191,8 @@ def doctest(context, py=None): # And check all the modules. for filename in glob(f"{_ROOT}/**/*.py", recursive=True): + if verbose: + print(f"doctesting {filename}") context.run(f"{get_py(py)} -m doctest {filename}") @@ -250,6 +260,6 @@ def autoflake(context, check=False, py=None): @task def format(context, check=False, py=None): """Run the formatters.""" - autoflake(context, check=check) - isort(context, check=check) - black(context, check=check) + autoflake(context, check=check, py=py) + isort(context, check=check, py=py) + black(context, check=check, py=py) diff --git a/tests/pandas_/test_mypy_dataframe.py b/tests/pandas_/test_mypy_dataframe.py index 90a2eb4..4cd57e8 100644 --- a/tests/pandas_/test_mypy_dataframe.py +++ b/tests/pandas_/test_mypy_dataframe.py @@ -5,9 +5,7 @@ class MyPyDataFrameTest(TestCase): - @skipUnless( - 7 < sys.version_info.minor < 11, "MyPy does not work with DataFrame on 3.7" - ) + @skipUnless(7 < sys.version_info.minor, "MyPy does not work with DataFrame on 3.7") def test_mypy_accepts_dataframe(self): exit_code, stdout, stderr = check_mypy_on_code( """ @@ -20,9 +18,7 @@ def test_mypy_accepts_dataframe(self): ) self.assertEqual(0, exit_code, stdout) - @skipUnless( - 7 < sys.version_info.minor < 11, "MyPy does not work with DataFrame on 3.7" - ) + @skipUnless(7 < sys.version_info.minor, "MyPy does not work with DataFrame on 3.7") def test_mypy_disapproves_dataframe_with_wrong_function_arguments(self): exit_code, stdout, stderr = check_mypy_on_code( """ @@ -43,9 +39,7 @@ def func(_: DataFrame[S["x: Float, y: Float"]]) -> None: self.assertIn('expected "DataFrame[Any]"', stdout) self.assertIn("Found 1 error in 1 file", stdout) - @skipUnless( - 7 < sys.version_info.minor < 11, "MyPy does not work with DataFrame on 3.7" - ) + @skipUnless(7 < sys.version_info.minor, "MyPy does not work with DataFrame on 3.7") def test_mypy_knows_of_dataframe_methods(self): # If MyPy knows of some arbitrary DataFrame methods, we can assume that # code completion works. diff --git a/tests/test_help_texts.py b/tests/test_help_texts.py new file mode 100644 index 0000000..b3f6d38 --- /dev/null +++ b/tests/test_help_texts.py @@ -0,0 +1,42 @@ +import pydoc +from unittest import TestCase + +from nptyping import ( + DataFrame, + Int, + NDArray, + RecArray, + Shape, + Structure, +) + + +class HelpTextsTest(TestCase): + def test_help_ndarray(self): + def func(arr: NDArray[Shape["2, 2"], Int]): + ... + + help_text = pydoc.render_doc(func) + + self.assertIn("arr: NDArray[Shape['2, 2'], Int]", help_text) + self.assertEqual("nptyping.ndarray", NDArray.__module__) + + def test_help_recdarray(self): + def func(arr: RecArray[Shape["2, 2"], Structure["[x, y]: Float"]]): + ... + + help_text = pydoc.render_doc(func) + + self.assertIn( + "arr: RecArray[Shape['2, 2'], Structure['[x, y]: Float']]", help_text + ) + self.assertEqual("nptyping.recarray", RecArray.__module__) + + def test_help_dataframe(self): + def func(df: DataFrame[Structure["[x, y]: Float"]]): + ... + + help_text = pydoc.render_doc(func) + + self.assertIn("df: DataFrame[Structure['[x, y]: Float']]", help_text) + self.assertEqual("nptyping.pandas_.dataframe", DataFrame.__module__) diff --git a/tests/test_structure_expression.py b/tests/test_structure_expression.py index 4c71a79..8ffa2e5 100644 --- a/tests/test_structure_expression.py +++ b/tests/test_structure_expression.py @@ -122,6 +122,7 @@ def test_validate_structure_expression_success(self): validate_structure_expression("abc: type, def: type") validate_structure_expression("abc: type[*, 2, ...], def: type[2 ]") validate_structure_expression("[abc, def]: type") + validate_structure_expression("[abc, def]: type, *") validate_structure_expression("[abc, def]: type[*, ...]") validate_structure_expression("[abc, def]: type1, ghi: type2") validate_structure_expression("[abc, def]: type1, [ghi, jkl]: type2") @@ -210,3 +211,18 @@ def test_create_name_to_type_dict(self): output = create_name_to_type_dict("a: t1, b: t2, c: t1") expected = {"a": "t1", "b": "t2", "c": "t1"} self.assertDictEqual(expected, output) + + def test_structure_depicting_at_least(self): + # Test that you can define a Structure that expresses a structure with + # at least some columns of some type. + dtype_true1 = np.dtype([("a", "U10"), ("b", "i4")]) + dtype_true2 = np.dtype([("a", "U10"), ("b", "i4"), ("c", "i4"), ("d", "i4")]) + dtype_false1 = np.dtype([("a", "U10"), ("b", "U10")]) + dtype_false2 = np.dtype([("b", "i4"), ("c", "i4"), ("d", "i4")]) + + structure = Structure["a: Str, b: Int32, *"] + + self.assertTrue(check_structure(dtype_true1, structure, dtype_per_name)) + self.assertTrue(check_structure(dtype_true2, structure, dtype_per_name)) + self.assertFalse(check_structure(dtype_false1, structure, dtype_per_name)) + self.assertFalse(check_structure(dtype_false2, structure, dtype_per_name)) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 77e0a84..8952694 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -47,6 +47,7 @@ "typing_.pyi", "pandas_/__init__.py", "pandas_/dataframe.py", + "pandas_/dataframe.pyi", "pandas_/typing_.py", } @@ -69,7 +70,8 @@ def working_dir(path: Path): os.chdir(origin) -@skipIf(sys.version_info.minor >= 11, "Does not work on 3.11 due to invoke") +# No need to run these tests on all versions. They take a long time. +@skipIf(sys.version_info.minor != 10, "Does not work on 3.11 due to invoke") class WheelTest(TestCase): temp_dir: TemporaryDirectory py: str @@ -111,6 +113,9 @@ def test_wheel_can_be_installed(self): subprocess.check_output( f"{self.py} -m ensurepip --upgrade --default-pip", shell=True ) + subprocess.check_output( + f"{self.py} -m pip install --upgrade pip", shell=True + ) subprocess.check_output( f"{self.pip} install {_ROOT / 'dist' / _WHEEL_NAME}", shell=True )