From b894d25d5c60b4b5b19d2a1c63360ca5051c532d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Wed, 27 Sep 2023 23:32:28 +0200 Subject: [PATCH 001/101] Disallow untyped defs in namedarray --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 25263928b20..89b3525a6db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,7 @@ module = [ # annotations — even with just `-> None` is sufficient for mypy to check them. [[tool.mypy.overrides]] disallow_untyped_defs = true -module = ["xarray.core.rolling_exp"] +module = ["xarray.core.rolling_exp", "xarray.namedarray.*", "xarray.tests.test_namedarray"] [tool.ruff] builtins = ["ellipsis"] From 895c2cb2f7e6f09eba43a3e76f7e0f3c3195f5cf Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Wed, 27 Sep 2023 23:36:59 +0200 Subject: [PATCH 002/101] Just use strict instead --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 89b3525a6db..fcc3b7859d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,11 @@ module = [ # annotations — even with just `-> None` is sufficient for mypy to check them. [[tool.mypy.overrides]] disallow_untyped_defs = true -module = ["xarray.core.rolling_exp", "xarray.namedarray.*", "xarray.tests.test_namedarray"] +module = ["xarray.core.rolling_exp"] + +[[tool.mypy.overrides]] +strict = true +module = ["xarray.namedarray.*", "xarray.tests.test_namedarray"] [tool.ruff] builtins = ["ellipsis"] From 15a2a8b5c551a082d3d9c63e014479669a5c7ff8 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Wed, 27 Sep 2023 23:46:21 +0200 Subject: [PATCH 003/101] Update pyproject.toml --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fcc3b7859d6..2788f329c75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,10 @@ module = [ "numpy.exceptions.*", # remove once support for `numpy<2.0` has been dropped ] +[[tool.mypy.overrides]] +strict = true +module = ["xarray.namedarray.*", "xarray.tests.test_namedarray"] + # Gradually we want to add more modules to this list, ratcheting up our total # coverage. Once a module is here, functions require annotations in order to # pass mypy. It would be especially useful to have tests here, because without @@ -127,10 +131,6 @@ module = [ disallow_untyped_defs = true module = ["xarray.core.rolling_exp"] -[[tool.mypy.overrides]] -strict = true -module = ["xarray.namedarray.*", "xarray.tests.test_namedarray"] - [tool.ruff] builtins = ["ellipsis"] exclude = [ From 89c8feaf3591123f2bcaa728d91b77ce71ca648d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 00:03:58 +0200 Subject: [PATCH 004/101] Test explicit list instead. --- pyproject.toml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2788f329c75..d920a4302b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,7 +119,35 @@ module = [ ] [[tool.mypy.overrides]] -strict = true +# Start off with these +warn_unused_configs = True +warn_redundant_casts = True +warn_unused_ignores = True + +# Getting these passing should be easy +strict_equality = True +strict_concatenate = True + +# Strongly recommend enabling this one as soon as you can +check_untyped_defs = True + +# These shouldn't be too much additional work, but may be tricky to +# get passing if you use a lot of untyped libraries +disallow_subclassing_any = True +disallow_untyped_decorators = True +disallow_any_generics = True + +# These next few are various gradations of forcing use of type annotations +disallow_untyped_calls = True +disallow_incomplete_defs = True +disallow_untyped_defs = True + +# This one isn't too hard to get passing, but return on investment is lower +no_implicit_reexport = True + +# This one can be tricky to get passing if you use a lot of untyped libraries +warn_return_any = True + module = ["xarray.namedarray.*", "xarray.tests.test_namedarray"] # Gradually we want to add more modules to this list, ratcheting up our total From 4e106502bc453a79ac671f46b49eee0f33038e93 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 00:08:51 +0200 Subject: [PATCH 005/101] Update pyproject.toml --- pyproject.toml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d920a4302b2..aa002fac074 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,33 +120,33 @@ module = [ [[tool.mypy.overrides]] # Start off with these -warn_unused_configs = True -warn_redundant_casts = True -warn_unused_ignores = True +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true # Getting these passing should be easy -strict_equality = True -strict_concatenate = True +strict_equality = true +strict_concatenate = true # Strongly recommend enabling this one as soon as you can -check_untyped_defs = True +check_untyped_defs = true # These shouldn't be too much additional work, but may be tricky to # get passing if you use a lot of untyped libraries -disallow_subclassing_any = True -disallow_untyped_decorators = True -disallow_any_generics = True +disallow_subclassing_any = true +disallow_untyped_decorators = true +disallow_any_generics = true # These next few are various gradations of forcing use of type annotations -disallow_untyped_calls = True -disallow_incomplete_defs = True -disallow_untyped_defs = True +disallow_untyped_calls = true +disallow_incomplete_defs = true +disallow_untyped_defs = true # This one isn't too hard to get passing, but return on investment is lower -no_implicit_reexport = True +no_implicit_reexport = true # This one can be tricky to get passing if you use a lot of untyped libraries -warn_return_any = True +warn_return_any = true module = ["xarray.namedarray.*", "xarray.tests.test_namedarray"] From fc7f69a8527af6533007d42caf9f6a5ac85ea24a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 00:27:31 +0200 Subject: [PATCH 006/101] Update pyproject.toml --- pyproject.toml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa002fac074..72fc1ec973f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,17 @@ module = [ "numpy.exceptions.*", # remove once support for `numpy<2.0` has been dropped ] +# Gradually we want to add more modules to this list, ratcheting up our total +# coverage. Once a module is here, functions require annotations in order to +# pass mypy. It would be especially useful to have tests here, because without +# annotating test functions, we don't have a great way of testing our type +# annotations — even with just `-> None` is sufficient for mypy to check them. +[[tool.mypy.overrides]] +disallow_untyped_defs = true +module = ["xarray.core.rolling_exp"] + +# Use strict = true whenever namedarray has become standalone. In the meantime +# don't forget to add all new files related to namedarray here: [[tool.mypy.overrides]] # Start off with these warn_unused_configs = true @@ -150,15 +161,6 @@ warn_return_any = true module = ["xarray.namedarray.*", "xarray.tests.test_namedarray"] -# Gradually we want to add more modules to this list, ratcheting up our total -# coverage. Once a module is here, functions require annotations in order to -# pass mypy. It would be especially useful to have tests here, because without -# annotating test functions, we don't have a great way of testing our type -# annotations — even with just `-> None` is sufficient for mypy to check them. -[[tool.mypy.overrides]] -disallow_untyped_defs = true -module = ["xarray.core.rolling_exp"] - [tool.ruff] builtins = ["ellipsis"] exclude = [ From 43f4e20c636230e7df4e380e0c2a26bb522df5fe Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 00:30:09 +0200 Subject: [PATCH 007/101] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 72fc1ec973f..2651eafce67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ module = ["xarray.core.rolling_exp"] # Use strict = true whenever namedarray has become standalone. In the meantime # don't forget to add all new files related to namedarray here: +# ref: https://mypy.readthedocs.io/en/stable/existing_code.html#introduce-stricter-options [[tool.mypy.overrides]] # Start off with these warn_unused_configs = true From 7cb3a09a897bb82186bc6f1eeccab760dc897b1f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:13:15 +0200 Subject: [PATCH 008/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 1495e111d85..609fd86d1eb 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -61,7 +61,7 @@ def is_duck_dask_array(x: typing.Any) -> bool: return is_duck_array(x) and is_dask_collection(x) -def to_0d_object_array(value: typing.Any) -> np.ndarray: +def to_0d_object_array(value: typing.Any) -> np.ndarray[object]: """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" result = np.empty((), dtype=object) result[()] = value From 4a83aa4445f66a289b18d4261adfee0cbf3f414a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:18:42 +0200 Subject: [PATCH 009/101] Update core.py --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 03bfa16682d..4c900a9820f 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -20,7 +20,7 @@ ) if typing.TYPE_CHECKING: - T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") + # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] From 7ad76349f6f5e60b23c497884e1bc03ce6b7a610 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:22:11 +0200 Subject: [PATCH 010/101] getmaskarray isn't typed yet --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 4c900a9820f..c1ba5316696 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -46,7 +46,7 @@ def as_compatible_data( return typing.cast(T_DuckArray, data) if isinstance(data, np.ma.MaskedArray): - mask = np.ma.getmaskarray(data) + mask = np.ma.getmaskarray(data) # type: ignore[no-untyped-call] if mask.any(): # TODO: requires refactoring/vendoring xarray.core.dtypes and xarray.core.duck_array_ops raise NotImplementedError("MaskedArray is not supported yet") From cebd1eb164f42e144765d87d028e4c4936d18fa9 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:28:19 +0200 Subject: [PATCH 011/101] Update core.py --- xarray/namedarray/core.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index c1ba5316696..5070f09fb63 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -74,13 +74,17 @@ class NamedArray(typing.Generic[T_DuckArray]): Numeric operations on this object implement array broadcasting and dimension alignment based on dimension names, rather than axis order.""" - __slots__ = ("_dims", "_data", "_attrs") + __slots__ = ("_data", "_dims", "_attrs") + + _data: T_DuckArray + _dims: Dims + _attrs: dict[typing.Any, typing.Any] | None def __init__( self, dims: DimsInput, data: T_DuckArray | np.typing.ArrayLike, - attrs: dict | None = None, + attrs: dict[typing.Any, typing.Any] | None = None, fastpath: bool = False, ): """ @@ -105,9 +109,9 @@ def __init__( """ - self._data: T_DuckArray = as_compatible_data(data, fastpath=fastpath) - self._dims: Dims = self._parse_dimensions(dims) - self._attrs: dict | None = dict(attrs) if attrs else None + self._data = as_compatible_data(data, fastpath=fastpath) + self._dims = self._parse_dimensions(dims) + self._attrs = dict(attrs) if attrs else None @property def ndim(self) -> int: @@ -208,7 +212,7 @@ def attrs(self) -> dict[typing.Any, typing.Any]: return self._attrs @attrs.setter - def attrs(self, value: Mapping) -> None: + def attrs(self, value: Mapping[typing.Any, typing.Any]) -> None: self._attrs = dict(value) def _check_shape(self, new_data: T_DuckArray) -> None: From 71a942b242bf41d227d9d756298cf0e4ef76ba66 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:59:53 +0200 Subject: [PATCH 012/101] add _Array protocol --- xarray/namedarray/core.py | 15 ++------------- xarray/namedarray/utils.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 5070f09fb63..290d3d5b1ef 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -13,6 +13,7 @@ from xarray.core.indexing import ExplicitlyIndexed from xarray.core.utils import Default, _default from xarray.namedarray.utils import ( + Self, T_DuckArray, is_duck_array, is_duck_dask_array, @@ -25,18 +26,6 @@ Dims = tuple[Hashable, ...] -try: - if sys.version_info >= (3, 11): - from typing import Self - else: - from typing_extensions import Self -except ImportError: - if typing.TYPE_CHECKING: - raise - else: - Self: typing.Any = None - - # TODO: Add tests! def as_compatible_data( data: T_DuckArray | np.typing.ArrayLike, fastpath: bool = False @@ -144,7 +133,7 @@ def __len__(self) -> int: raise TypeError("len() of unsized object") from exc @property - def dtype(self) -> np.dtype: + def dtype(self) -> np.dtype[typing.Any]: """ Data-type of the array’s elements. diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 609fd86d1eb..b3feb75c55c 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -6,6 +6,18 @@ import numpy as np +try: + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self +except ImportError: + if typing.TYPE_CHECKING: + raise + else: + Self: typing.Any = None + + if typing.TYPE_CHECKING: if sys.version_info >= (3, 10): from typing import TypeGuard @@ -14,7 +26,28 @@ # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DuckArray = typing.TypeVar("T_DuckArray", bound=typing.Any) +T_DType = typing.TypeVar("T_DType", bounds=np.dtype) + + +class _Array(typing.Protocol[T_DType]): + @property + def dtype(self) -> T_DType: + ... + + @property + def shape(self) -> tuple[int, ...]: + ... + + @property + def real(self) -> Self: + ... + + @property + def imag(self) -> Self: + ... + + +T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array) def module_available(module: str) -> bool: From 4cdbed5cf1d2af1c20234a1f2a7d988ceeb52399 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 18:00:40 +0000 Subject: [PATCH 013/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 290d3d5b1ef..f185f542548 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -2,7 +2,6 @@ import copy import math -import sys import typing from collections.abc import Hashable, Iterable, Mapping From c80ff3047043b48760e21fe1d4cae354166aeb98 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 20:02:50 +0200 Subject: [PATCH 014/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index b3feb75c55c..3d80ad237ea 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -26,7 +26,7 @@ # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DType = typing.TypeVar("T_DType", bounds=np.dtype) +T_DType = typing.TypeVar("T_DType", bound=np.dtype) class _Array(typing.Protocol[T_DType]): From f6c6f44de2af2e50c85f7d237b72c62d93d1b8b4 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 20:45:59 +0200 Subject: [PATCH 015/101] Update utils.py --- xarray/namedarray/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 3d80ad237ea..8e3ec7fc6d7 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -26,12 +26,14 @@ # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DType = typing.TypeVar("T_DType", bound=np.dtype) +T_DType_co = typing.TypeVar( + "T_DType", bound=np.dtype[np.typing.DTypeLike], covariant=True +) -class _Array(typing.Protocol[T_DType]): +class _Array(typing.Protocol[T_DType_co]): @property - def dtype(self) -> T_DType: + def dtype(self) -> T_DType_co: ... @property From 4b897eb732787a78bef3005de627bb28805d4cd8 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 21:37:11 +0200 Subject: [PATCH 016/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 8e3ec7fc6d7..35f30eca631 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -27,7 +27,7 @@ # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more T_DType_co = typing.TypeVar( - "T_DType", bound=np.dtype[np.typing.DTypeLike], covariant=True + "T_DType_co", bound=np.dtype[np.typing.DTypeLike], covariant=True ) From 7081ee106b076c62d42998f2e7341ed46a3dd2ba Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 21:57:36 +0200 Subject: [PATCH 017/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 35f30eca631..ac57b880a75 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -96,7 +96,7 @@ def is_duck_dask_array(x: typing.Any) -> bool: return is_duck_array(x) and is_dask_collection(x) -def to_0d_object_array(value: typing.Any) -> np.ndarray[object]: +def to_0d_object_array(value: typing.Any) -> np.ndarray[typing.Any, object]: """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" result = np.empty((), dtype=object) result[()] = value From 87958d9c001187dac75b4f86494b5b5546e1f82f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:05:57 +0200 Subject: [PATCH 018/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index ac57b880a75..59bef318631 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -96,7 +96,7 @@ def is_duck_dask_array(x: typing.Any) -> bool: return is_duck_array(x) and is_dask_collection(x) -def to_0d_object_array(value: typing.Any) -> np.ndarray[typing.Any, object]: +def to_0d_object_array(value: typing.Any) -> np.ndarray[typing.Any, np.object_]: """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" result = np.empty((), dtype=object) result[()] = value From f913206b75935d37bbd0b87a0dcc3e4582eeaf43 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:21:08 +0200 Subject: [PATCH 019/101] Update utils.py --- xarray/namedarray/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 59bef318631..f9c4de0555b 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -24,11 +24,9 @@ else: from typing_extensions import TypeGuard -# temporary placeholder for indicating an array api compliant type. -# hopefully in the future we can narrow this down more -T_DType_co = typing.TypeVar( - "T_DType_co", bound=np.dtype[np.typing.DTypeLike], covariant=True -) + +# https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array +T_DType_co = typing.TypeVar("T_DType_co", bound=np.generic, covariant=True) class _Array(typing.Protocol[T_DType_co]): @@ -49,6 +47,8 @@ def imag(self) -> Self: ... +# temporary placeholder for indicating an array api compliant type. +# hopefully in the future we can narrow this down more T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array) From 3b0c122f2f54cc16dd6f3e98fedf4ee7b457b296 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:29:37 +0200 Subject: [PATCH 020/101] Update utils.py --- xarray/namedarray/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index f9c4de0555b..0a7f473e83f 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -96,7 +96,9 @@ def is_duck_dask_array(x: typing.Any) -> bool: return is_duck_array(x) and is_dask_collection(x) -def to_0d_object_array(value: typing.Any) -> np.ndarray[typing.Any, np.object_]: +def to_0d_object_array( + value: typing.Any, +) -> np.ndarray[typing.Any, np.dtype[np.object_]]: """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" result = np.empty((), dtype=object) result[()] = value From 027f300c2cb8f584117dd6dfe36c819b3fdfae74 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:33:59 +0200 Subject: [PATCH 021/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 0a7f473e83f..a9f2fd0161d 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -49,7 +49,7 @@ def imag(self) -> Self: # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array) +T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[T_DType_co]) def module_available(module: str) -> bool: From 23ec9fe97206d4a0a65945deaa4ff9ba1baf2c2c Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:51:55 +0200 Subject: [PATCH 022/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index a9f2fd0161d..9feab1006ed 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -49,7 +49,7 @@ def imag(self) -> Self: # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[T_DType_co]) +T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.generic]) def module_available(module: str) -> bool: From d94b7669dab8417a3f718de589bf497e7db4ed50 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:59:25 +0200 Subject: [PATCH 023/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 9feab1006ed..a33a3f784b1 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -49,7 +49,7 @@ def imag(self) -> Self: # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.generic]) +T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.dtype]) def module_available(module: str) -> bool: From c353336a99b04de0a86284462ae1d4755732404d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 23:04:46 +0200 Subject: [PATCH 024/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index a33a3f784b1..9dd1e97ad08 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -26,7 +26,7 @@ # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array -T_DType_co = typing.TypeVar("T_DType_co", bound=np.generic, covariant=True) +T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) class _Array(typing.Protocol[T_DType_co]): From 5e26ebaa47dafb605bd44df71657a85eaca5a967 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 23:14:18 +0200 Subject: [PATCH 025/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 36 +++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 9d37a6c794c..8913a914350 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import Any + import numpy as np import pytest @@ -59,7 +63,8 @@ class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): def test_properties() -> None: data = 0.5 * np.arange(10).reshape(2, 5) - named_array: NamedArray[np.ndarray] = NamedArray(["x", "y"], data, {"key": "value"}) + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray(["x", "y"], data, {"key": "value"}) assert named_array.dims == ("x", "y") assert np.array_equal(named_array.data, data) assert named_array.attrs == {"key": "value"} @@ -71,9 +76,8 @@ def test_properties() -> None: def test_attrs() -> None: - named_array: NamedArray[np.ndarray] = NamedArray( - ["x", "y"], np.arange(10).reshape(2, 5) - ) + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray(["x", "y"], np.arange(10).reshape(2, 5)) assert named_array.attrs == {} named_array.attrs["key"] = "value" assert named_array.attrs == {"key": "value"} @@ -82,7 +86,8 @@ def test_attrs() -> None: def test_data(random_inputs) -> None: - named_array: NamedArray[np.ndarray] = NamedArray(["x", "y", "z"], random_inputs) + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray(["x", "y", "z"], random_inputs) assert np.array_equal(named_array.data, random_inputs) with pytest.raises(ValueError): named_array.data = np.random.random((3, 4)).astype(np.float64) @@ -96,8 +101,9 @@ def test_data(random_inputs) -> None: (np.bytes_("foo"), np.dtype("S3")), ], ) -def test_0d_string(data, dtype: np.typing.DTypeLike) -> None: - named_array: NamedArray[np.ndarray] = NamedArray([], data) +def test_0d_string(data: Any, dtype: np.typing.DTypeLike) -> None: + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray([], data) assert named_array.data == data assert named_array.dims == () assert named_array.sizes == {} @@ -108,7 +114,8 @@ def test_0d_string(data, dtype: np.typing.DTypeLike) -> None: def test_0d_object() -> None: - named_array: NamedArray[np.ndarray] = NamedArray([], (10, 12, 12)) + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray([], (10, 12, 12)) expected_data = np.empty((), dtype=object) expected_data[()] = (10, 12, 12) assert np.array_equal(named_array.data, expected_data) @@ -122,7 +129,8 @@ def test_0d_object() -> None: def test_0d_datetime() -> None: - named_array: NamedArray[np.ndarray] = NamedArray([], np.datetime64("2000-01-01")) + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray([], np.datetime64("2000-01-01")) assert named_array.dtype == np.dtype("datetime64[D]") @@ -140,8 +148,9 @@ def test_0d_datetime() -> None: (np.timedelta64(1, "as"), np.dtype("timedelta64[as]")), ], ) -def test_0d_timedelta(timedelta, expected_dtype: np.dtype) -> None: - named_array: NamedArray[np.ndarray] = NamedArray([], timedelta) +def test_0d_timedelta(timedelta: np.timedelta, expected_dtype: np.timedelta64) -> None: + named_array: NamedArray[np.ndarray[np.timedelta64]] + named_array = NamedArray([], timedelta) assert named_array.dtype == expected_dtype assert named_array.data == timedelta @@ -156,8 +165,9 @@ def test_0d_timedelta(timedelta, expected_dtype: np.dtype) -> None: ([], [], ("x",), True), ], ) -def test_dims_setter(dims, data_shape, new_dims, raises: bool) -> None: - named_array: NamedArray[np.ndarray] = NamedArray(dims, np.random.random(data_shape)) +def test_dims_setter(dims: Any, data_shape: Any, new_dims: Any, raises: bool) -> None: + named_array: NamedArray[np.ndarray[Any]] + named_array = NamedArray(dims, np.random.random(data_shape)) assert named_array.dims == tuple(dims) if raises: with pytest.raises(ValueError): From c5a95944cbbb5f7b4685706b10e0b23647111605 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 23:19:34 +0200 Subject: [PATCH 026/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 9dd1e97ad08..d3369237fac 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -49,7 +49,7 @@ def imag(self) -> Self: # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.dtype]) +T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.dtype[np.generic]]) def module_available(module: str) -> bool: From ecb50c07101f232d35cb47d04f5512e74ec1c629 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Thu, 28 Sep 2023 23:23:15 +0200 Subject: [PATCH 027/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 8913a914350..ce7af35f389 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -63,7 +63,7 @@ class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): def test_properties() -> None: data = 0.5 * np.arange(10).reshape(2, 5) - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray(["x", "y"], data, {"key": "value"}) assert named_array.dims == ("x", "y") assert np.array_equal(named_array.data, data) @@ -76,7 +76,7 @@ def test_properties() -> None: def test_attrs() -> None: - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray(["x", "y"], np.arange(10).reshape(2, 5)) assert named_array.attrs == {} named_array.attrs["key"] = "value" @@ -86,7 +86,7 @@ def test_attrs() -> None: def test_data(random_inputs) -> None: - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray(["x", "y", "z"], random_inputs) assert np.array_equal(named_array.data, random_inputs) with pytest.raises(ValueError): @@ -102,7 +102,7 @@ def test_data(random_inputs) -> None: ], ) def test_0d_string(data: Any, dtype: np.typing.DTypeLike) -> None: - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray([], data) assert named_array.data == data assert named_array.dims == () @@ -114,7 +114,7 @@ def test_0d_string(data: Any, dtype: np.typing.DTypeLike) -> None: def test_0d_object() -> None: - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray([], (10, 12, 12)) expected_data = np.empty((), dtype=object) expected_data[()] = (10, 12, 12) @@ -129,7 +129,7 @@ def test_0d_object() -> None: def test_0d_datetime() -> None: - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray([], np.datetime64("2000-01-01")) assert named_array.dtype == np.dtype("datetime64[D]") @@ -149,7 +149,7 @@ def test_0d_datetime() -> None: ], ) def test_0d_timedelta(timedelta: np.timedelta, expected_dtype: np.timedelta64) -> None: - named_array: NamedArray[np.ndarray[np.timedelta64]] + named_array: NamedArray[np.ndarray[Any, np.timedelta64]] named_array = NamedArray([], timedelta) assert named_array.dtype == expected_dtype assert named_array.data == timedelta @@ -166,7 +166,7 @@ def test_0d_timedelta(timedelta: np.timedelta, expected_dtype: np.timedelta64) - ], ) def test_dims_setter(dims: Any, data_shape: Any, new_dims: Any, raises: bool) -> None: - named_array: NamedArray[np.ndarray[Any]] + named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray(dims, np.random.random(data_shape)) assert named_array.dims == tuple(dims) if raises: From 9ab9dae322ae23b5c763751d971ee4ac6acd53b8 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 29 Sep 2023 00:09:03 +0200 Subject: [PATCH 028/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index ce7af35f389..937d0534dce 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -7,7 +7,7 @@ import xarray as xr from xarray.namedarray.core import NamedArray, as_compatible_data -from xarray.namedarray.utils import T_DuckArray +from xarray.namedarray.utils import Self, T_DuckArray @pytest.fixture @@ -49,6 +49,22 @@ class CustomArray(xr.core.indexing.NDArrayMixin): def __init__(self, array): self.array = array + @property + def dtype(self) -> np.dtype[np.generic]: + raise NotImplementedError + + @property + def shape(self) -> tuple[int, ...]: + raise NotImplementedError + + @property + def real(self) -> Self: + raise NotImplementedError + + @property + def imag(self) -> Self: + raise NotImplementedError + class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): pass @@ -148,8 +164,10 @@ def test_0d_datetime() -> None: (np.timedelta64(1, "as"), np.dtype("timedelta64[as]")), ], ) -def test_0d_timedelta(timedelta: np.timedelta, expected_dtype: np.timedelta64) -> None: - named_array: NamedArray[np.ndarray[Any, np.timedelta64]] +def test_0d_timedelta( + timedelta: np.timedelta64, expected_dtype: np.dtype[np.timedelta64] +) -> None: + named_array: NamedArray[np.ndarray[Any, np.dtype[np.timedelta64]]] named_array = NamedArray([], timedelta) assert named_array.dtype == expected_dtype assert named_array.data == timedelta From 41bd67c75dc40941a210ec0611ee9ab0cdaccc35 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Fri, 29 Sep 2023 18:59:50 +0200 Subject: [PATCH 029/101] Update utils.py --- xarray/namedarray/utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index d3369237fac..f95b8644933 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -8,15 +8,16 @@ try: if sys.version_info >= (3, 11): - from typing import Self + from typing import Self as _Self else: - from typing_extensions import Self + from typing_extensions import Self as _Self except ImportError: if typing.TYPE_CHECKING: raise else: - Self: typing.Any = None + _Self: typing.Any = None +Self = _Self if typing.TYPE_CHECKING: if sys.version_info >= (3, 10): @@ -27,6 +28,7 @@ # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) +# T_DType = typing.TypeVar("T_DType", bound=np.dtype[np.generic]) class _Array(typing.Protocol[T_DType_co]): @@ -46,6 +48,11 @@ def real(self) -> Self: def imag(self) -> Self: ... + # def __array__( + # self, dtype: np.typing.DTypeLike = None + # ) -> np.ndarray[typing.Any, np.dtype[np.generic]]: + # ... + # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more From 5f58cee4dd2ec031587ace19aede6908cd18ff6a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:06:47 +0200 Subject: [PATCH 030/101] Update utils.py --- xarray/namedarray/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index f95b8644933..14269794c05 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -11,13 +11,18 @@ from typing import Self as _Self else: from typing_extensions import Self as _Self + + if sys.version_info >= (3, 10): + from typing import TypeAlias + else: + from typing_extensions import TypeAlias except ImportError: if typing.TYPE_CHECKING: raise else: _Self: typing.Any = None -Self = _Self +Self: TypeAlias = _Self if typing.TYPE_CHECKING: if sys.version_info >= (3, 10): From 7f1a94e3f6ff120aa7f8e025827b9c483140aeb3 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:30:03 +0200 Subject: [PATCH 031/101] Update utils.py --- xarray/namedarray/utils.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 14269794c05..005034a685f 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -6,23 +6,6 @@ import numpy as np -try: - if sys.version_info >= (3, 11): - from typing import Self as _Self - else: - from typing_extensions import Self as _Self - - if sys.version_info >= (3, 10): - from typing import TypeAlias - else: - from typing_extensions import TypeAlias -except ImportError: - if typing.TYPE_CHECKING: - raise - else: - _Self: typing.Any = None - -Self: TypeAlias = _Self if typing.TYPE_CHECKING: if sys.version_info >= (3, 10): @@ -30,6 +13,10 @@ else: from typing_extensions import TypeGuard + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) From 685ca7cf270a2e5abf3741758c0b09985d33f56c Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:30:12 +0200 Subject: [PATCH 032/101] Update core.py --- xarray/namedarray/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index f185f542548..6bdcb4ff648 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -12,7 +12,6 @@ from xarray.core.indexing import ExplicitlyIndexed from xarray.core.utils import Default, _default from xarray.namedarray.utils import ( - Self, T_DuckArray, is_duck_array, is_duck_dask_array, @@ -20,6 +19,8 @@ ) if typing.TYPE_CHECKING: + from xarray.namedarray.utils import Self + # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] From 7aa2f5752a7fb4580db9bcc23a9a73dd555b4476 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 30 Sep 2023 06:30:58 +0000 Subject: [PATCH 033/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 005034a685f..ab081a44b11 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -6,7 +6,6 @@ import numpy as np - if typing.TYPE_CHECKING: if sys.version_info >= (3, 10): from typing import TypeGuard From 99b0acab1117b5f47876d47b44a1c1b995b5cddd Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:58:49 +0200 Subject: [PATCH 034/101] Update utils.py --- xarray/namedarray/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index ab081a44b11..daf6ae6f547 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -16,7 +16,8 @@ from typing import Self else: from typing_extensions import Self - +else: + Self: typing.Any = None # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) # T_DType = typing.TypeVar("T_DType", bound=np.dtype[np.generic]) From 84b689447e8f6ad10eb799c427f2c08d652d4297 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 09:10:35 +0200 Subject: [PATCH 035/101] Update core.py --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 6bdcb4ff648..19b9a2a4c64 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -408,7 +408,7 @@ def _nonzero(self) -> tuple[Self, ...]: def _as_sparse( self, sparse_format: str | Default = _default, - fill_value=dtypes.NA, + fill_value: typing.Any = dtypes.NA, ) -> Self: """ use sparse-array as backend. From 08d11ef2c256e95e4f4973d1363369a0b0dfbd46 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 09:10:39 +0200 Subject: [PATCH 036/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 937d0534dce..bb01f2cbd72 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -11,7 +11,7 @@ @pytest.fixture -def random_inputs() -> np.ndarray: +def random_inputs() -> np.ndarray[Any, np.float32]: return np.arange(3 * 4 * 5, dtype=np.float32).reshape((3, 4, 5)) @@ -32,7 +32,7 @@ def test_as_compatible_data( def test_as_compatible_data_with_masked_array() -> None: - masked_array = np.ma.array([1, 2, 3], mask=[False, True, False]) + masked_array = np.ma.array([1, 2, 3], mask=[False, True, False]) # type: ignore[no-untyped-call] with pytest.raises(NotImplementedError): as_compatible_data(masked_array) @@ -43,7 +43,9 @@ def test_as_compatible_data_with_0d_object() -> None: np.array_equal(as_compatible_data(data), data) -def test_as_compatible_data_with_explicitly_indexed(random_inputs) -> None: +def test_as_compatible_data_with_explicitly_indexed( + random_inputs: np.ndarray[Any, Any] +) -> None: # TODO: Make xr.core.indexing.ExplicitlyIndexed pass is_duck_array and remove this test. class CustomArray(xr.core.indexing.NDArrayMixin): def __init__(self, array): @@ -101,7 +103,7 @@ def test_attrs() -> None: assert named_array.attrs == {"key": "value2"} -def test_data(random_inputs) -> None: +def test_data(random_inputs: np.ndarray[Any, Any]) -> None: named_array: NamedArray[np.ndarray[Any, Any]] named_array = NamedArray(["x", "y", "z"], random_inputs) assert np.array_equal(named_array.data, random_inputs) From 196a5c6b01e754e6acf4d4d3ad9a479951ee8467 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 09:15:40 +0200 Subject: [PATCH 037/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index bb01f2cbd72..5b43bf1cafd 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -11,7 +11,7 @@ @pytest.fixture -def random_inputs() -> np.ndarray[Any, np.float32]: +def random_inputs() -> np.ndarray[Any, np.dtype[np.float32]]: return np.arange(3 * 4 * 5, dtype=np.float32).reshape((3, 4, 5)) From 07e308544465df12d8fb782e0f863f0f9560151a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 09:20:32 +0200 Subject: [PATCH 038/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 5b43bf1cafd..851cdb94150 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -48,7 +48,7 @@ def test_as_compatible_data_with_explicitly_indexed( ) -> None: # TODO: Make xr.core.indexing.ExplicitlyIndexed pass is_duck_array and remove this test. class CustomArray(xr.core.indexing.NDArrayMixin): - def __init__(self, array): + def __init__(self, array: T_DuckArray) -> None: self.array = array @property From 707f2442cda40085aacf552ee3332380a21368e0 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 09:35:47 +0200 Subject: [PATCH 039/101] Update utils.py --- xarray/namedarray/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index daf6ae6f547..37d0e2c2340 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +from enum import Enum + import importlib import sys import typing @@ -51,6 +53,14 @@ def imag(self) -> Self: T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.dtype[np.generic]]) +# Singleton type, as per https://github.com/python/typing/pull/240 +class Default(Enum): + token: typing.Literal[0] = 0 + + +_default = Default.token + + def module_available(module: str) -> bool: """Checks whether a module is installed without importing it. From a3901bc41b80cbcbf1544e063f974f4b5f234003 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 09:35:52 +0200 Subject: [PATCH 040/101] Update core.py --- xarray/namedarray/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 19b9a2a4c64..66050f4ef53 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -10,8 +10,9 @@ # TODO: get rid of this after migrating this class to array API from xarray.core import dtypes from xarray.core.indexing import ExplicitlyIndexed -from xarray.core.utils import Default, _default from xarray.namedarray.utils import ( + Default, + _default, T_DuckArray, is_duck_array, is_duck_dask_array, From df13d4770d62485a6a6196d7bf87a07479687032 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 30 Sep 2023 07:36:30 +0000 Subject: [PATCH 041/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 2 +- xarray/namedarray/utils.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 66050f4ef53..84d1b16ddd0 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -12,8 +12,8 @@ from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.utils import ( Default, - _default, T_DuckArray, + _default, is_duck_array, is_duck_dask_array, to_0d_object_array, diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 37d0e2c2340..da1efbfa40c 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -1,10 +1,9 @@ from __future__ import annotations -from enum import Enum - import importlib import sys import typing +from enum import Enum import numpy as np From f76aeb11085170be91a3b19a218c605dfab8f266 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 10:03:17 +0200 Subject: [PATCH 042/101] Update core.py --- xarray/namedarray/core.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 66050f4ef53..dbf3bdaff0b 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -25,12 +25,12 @@ # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] + DataInput = typing.Union[T_DuckArray, np.typing.ArrayLike] + AttrsInput = typing.Union[Mapping[typing.Any, typing.Any], None] # TODO: Add tests! -def as_compatible_data( - data: T_DuckArray | np.typing.ArrayLike, fastpath: bool = False -) -> T_DuckArray: +def as_compatible_data(data: DataInput, fastpath: bool = False) -> T_DuckArray: if fastpath and getattr(data, "ndim", 0) > 0: # can't use fastpath (yet) for scalars return typing.cast(T_DuckArray, data) @@ -73,8 +73,8 @@ class NamedArray(typing.Generic[T_DuckArray]): def __init__( self, dims: DimsInput, - data: T_DuckArray | np.typing.ArrayLike, - attrs: dict[typing.Any, typing.Any] | None = None, + data: DataInput, + attrs: AttrsInput = None, fastpath: bool = False, ): """ @@ -223,7 +223,7 @@ def data(self) -> T_DuckArray: return self._data @data.setter - def data(self, data: T_DuckArray | np.typing.ArrayLike) -> None: + def data(self, data: DataInput) -> None: data = as_compatible_data(data) self._check_shape(data) self._data = data @@ -332,7 +332,12 @@ def sizes(self) -> dict[Hashable, int]: """Ordered mapping from dimension names to lengths.""" return dict(zip(self.dims, self.shape)) - def _replace(self, dims=_default, data=_default, attrs=_default) -> Self: + def _replace( + self, + dims: DimsInput | Default = _default, + data: DataInput | Default = _default, + attrs: AttrsInput | Default = _default, + ) -> Self: if dims is _default: dims = copy.copy(self._dims) if data is _default: @@ -344,7 +349,7 @@ def _replace(self, dims=_default, data=_default, attrs=_default) -> Self: def _copy( self, deep: bool = True, - data: T_DuckArray | np.typing.ArrayLike | None = None, + data: DataInput | None = None, memo: dict[int, typing.Any] | None = None, ) -> Self: if data is None: @@ -370,7 +375,7 @@ def __deepcopy__(self, memo: dict[int, typing.Any] | None = None) -> Self: def copy( self, deep: bool = True, - data: T_DuckArray | np.typing.ArrayLike | None = None, + data: DataInput | None = None, ) -> Self: """Returns a copy of this object. From 386526424c38879c69e1a922327568cd4156831a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 10:10:07 +0200 Subject: [PATCH 043/101] Update core.py --- xarray/namedarray/core.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 5c33b9b4276..154585db8fb 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -25,12 +25,13 @@ # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] - DataInput = typing.Union[T_DuckArray, np.typing.ArrayLike] AttrsInput = typing.Union[Mapping[typing.Any, typing.Any], None] # TODO: Add tests! -def as_compatible_data(data: DataInput, fastpath: bool = False) -> T_DuckArray: +def as_compatible_data( + data: T_DuckArray | np.typing.ArrayLike, fastpath: bool = False +) -> T_DuckArray: if fastpath and getattr(data, "ndim", 0) > 0: # can't use fastpath (yet) for scalars return typing.cast(T_DuckArray, data) @@ -73,7 +74,7 @@ class NamedArray(typing.Generic[T_DuckArray]): def __init__( self, dims: DimsInput, - data: DataInput, + data: T_DuckArray | np.typing.ArrayLike, attrs: AttrsInput = None, fastpath: bool = False, ): @@ -223,7 +224,7 @@ def data(self) -> T_DuckArray: return self._data @data.setter - def data(self, data: DataInput) -> None: + def data(self, data: T_DuckArray | np.typing.ArrayLike) -> None: data = as_compatible_data(data) self._check_shape(data) self._data = data @@ -335,7 +336,7 @@ def sizes(self) -> dict[Hashable, int]: def _replace( self, dims: DimsInput | Default = _default, - data: DataInput | Default = _default, + data: T_DuckArray | np.typing.ArrayLike | Default = _default, attrs: AttrsInput | Default = _default, ) -> Self: if dims is _default: @@ -349,7 +350,7 @@ def _replace( def _copy( self, deep: bool = True, - data: DataInput | None = None, + data: T_DuckArray | np.typing.ArrayLike | None = None, memo: dict[int, typing.Any] | None = None, ) -> Self: if data is None: @@ -375,7 +376,7 @@ def __deepcopy__(self, memo: dict[int, typing.Any] | None = None) -> Self: def copy( self, deep: bool = True, - data: DataInput | None = None, + data: T_DuckArray | np.typing.ArrayLike | None = None, ) -> Self: """Returns a copy of this object. From b61d9a851b5641706e0dfd89d487ed4ff424a33d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 30 Sep 2023 10:49:08 +0200 Subject: [PATCH 044/101] Update utils.py --- xarray/namedarray/utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index da1efbfa40c..9be306ce829 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -17,8 +17,15 @@ from typing import Self else: from typing_extensions import Self -else: - Self: typing.Any = None + + try: + from dask.array import Array as DaskArray + from dask.types import DaskCollection + except ImportError: + DaskArray = np.ndarray # type: ignore + DaskCollection: np.Any = np.ndarray # type: ignore + + # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) # T_DType = typing.TypeVar("T_DType", bound=np.dtype[np.generic]) @@ -78,11 +85,11 @@ def module_available(module: str) -> bool: return importlib.util.find_spec(module) is not None -def is_dask_collection(x: typing.Any) -> bool: +def is_dask_collection(x: typing.Any) -> TypeGuard[DaskCollection]: if module_available("dask"): - from dask.base import is_dask_collection + from dask.typing import DaskCollection - return is_dask_collection(x) + return isinstance(x, DaskCollection) return False @@ -100,7 +107,7 @@ def is_duck_array(value: typing.Any) -> TypeGuard[T_DuckArray]: ) -def is_duck_dask_array(x: typing.Any) -> bool: +def is_duck_dask_array(x: typing.Any) -> TypeGuard[DaskArray]: return is_duck_array(x) and is_dask_collection(x) From 4dec3cafde2ba68f9b3a1705bb6dff692f96f9a2 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 19:01:19 +0200 Subject: [PATCH 045/101] Update core.py --- xarray/namedarray/core.py | 83 ++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 154585db8fb..624bfb6f67e 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -22,6 +22,17 @@ if typing.TYPE_CHECKING: from xarray.namedarray.utils import Self + try: + from dask.typing import ( + SchedulerGetCallable, + PostComputeCallable, + PostPersistCallable, + ) + except ImportError: + SchedulerGetCallable: typing.Any + PostComputeCallable: typing.Any + PostPersistCallable: typing.Any + # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] @@ -251,45 +262,73 @@ def imag(self) -> Self: """ return self._replace(data=self.data.imag) - def __dask_tokenize__(self): - # Use v.data, instead of v._data, in order to cope with the wrappers - # around NetCDF and the like - from dask.base import normalize_token + def __dask_tokenize__(self) -> Hashable: + if is_duck_dask_array(self._data): + # Use v.data, instead of v._data, in order to cope with the wrappers + # around NetCDF and the like + from dask.base import normalize_token - return normalize_token((type(self), self._dims, self.data, self.attrs)) + return normalize_token((type(self), self._dims, self.data, self.attrs)) + else: + raise NotImplementedError("Method requires self.data to be a dask array") - def __dask_graph__(self): - return self._data.__dask_graph__() if is_duck_dask_array(self._data) else None + def __dask_graph__(self) -> Mapping[typing.Any, typing.Any]: + if is_duck_dask_array(self._data): + return self._data.__dask_graph__() + else: + # raise NotImplementedError("Method requires self.data to be a dask array") + return None - def __dask_keys__(self): - return self._data.__dask_keys__() + def __dask_keys__(self) -> list[Hashable]: + if is_duck_dask_array(self._data): + return self._data.__dask_keys__() + else: + raise NotImplementedError("Method requires self.data to be a dask array") - def __dask_layers__(self): - return self._data.__dask_layers__() + def __dask_layers__(self) -> typing.Sequence[str]: + if is_duck_dask_array(self._data): + return self._data.__dask_layers__() + else: + raise NotImplementedError("Method requires self.data to be a dask array") @property def __dask_optimize__(self) -> typing.Callable: - return self._data.__dask_optimize__ + if is_duck_dask_array(self._data): + return self._data.__dask_optimize__() + else: + raise NotImplementedError("Method requires self.data to be a dask array") @property - def __dask_scheduler__(self) -> typing.Callable: - return self._data.__dask_scheduler__ + def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable]: + if is_duck_dask_array(self._data): + return self._data.__dask_scheduler__() + else: + raise NotImplementedError("Method requires self.data to be a dask array") def __dask_postcompute__( self, - ) -> tuple[typing.Callable, tuple[typing.Any, ...]]: - array_func, array_args = self._data.__dask_postcompute__() - return self._dask_finalize, (array_func,) + array_args + ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]]: + if is_duck_dask_array(self._data): + array_func, array_args = self._data.__dask_postcompute__() + return self._dask_finalize, (array_func,) + array_args + else: + raise NotImplementedError("Method requires self.data to be a dask array") def __dask_postpersist__( self, - ) -> tuple[typing.Callable, tuple[typing.Any, ...]]: - array_func, array_args = self._data.__dask_postpersist__() - return self._dask_finalize, (array_func,) + array_args + ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]]: + if is_duck_dask_array(self._data): + array_func, array_args = self._data.__dask_postpersist__() + return self._dask_finalize, (array_func,) + array_args + else: + raise NotImplementedError("Method requires self.data to be a dask array") def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self: - data = array_func(results, *args, **kwargs) - return type(self)(self._dims, data, attrs=self._attrs) + if is_duck_dask_array(self._data): + data = array_func(results, *args, **kwargs) + return type(self)(self._dims, data, attrs=self._attrs) + else: + raise NotImplementedError("Method requires self.data to be a dask array") @property def chunks(self) -> tuple[tuple[int, ...], ...] | None: From 26ac902554e13643f4a3b59c8b83b20597676173 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 17:02:03 +0000 Subject: [PATCH 046/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 624bfb6f67e..dc58f287039 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -24,9 +24,9 @@ try: from dask.typing import ( - SchedulerGetCallable, PostComputeCallable, PostPersistCallable, + SchedulerGetCallable, ) except ImportError: SchedulerGetCallable: typing.Any From 9d232451ae5a735028bd85e6cecdb733f1d2de8f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 19:40:21 +0200 Subject: [PATCH 047/101] Update core.py --- xarray/namedarray/core.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 624bfb6f67e..7394530a82b 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -24,14 +24,18 @@ try: from dask.typing import ( + Graph, + NestedKeys, SchedulerGetCallable, PostComputeCallable, PostPersistCallable, ) except ImportError: - SchedulerGetCallable: typing.Any - PostComputeCallable: typing.Any - PostPersistCallable: typing.Any + Graph: typing.Any # typ: ignore[no-redef] + NestedKeys: typing.Any # typ: ignore[no-redef] + SchedulerGetCallable: typing.Any # typ: ignore[no-redef] + PostComputeCallable: typing.Any # typ: ignore[no-redef] + PostPersistCallable: typing.Any # typ: ignore[no-redef] # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] @@ -268,18 +272,21 @@ def __dask_tokenize__(self) -> Hashable: # around NetCDF and the like from dask.base import normalize_token - return normalize_token((type(self), self._dims, self.data, self.attrs)) + return normalize_token( + (type(self), self._dims, self.data, self.attrs) + ) # type: ignore[no-any-return] else: raise NotImplementedError("Method requires self.data to be a dask array") - def __dask_graph__(self) -> Mapping[typing.Any, typing.Any]: + def __dask_graph__(self) -> Graph: if is_duck_dask_array(self._data): return self._data.__dask_graph__() else: + # TODO: Should this method just raise instead? # raise NotImplementedError("Method requires self.data to be a dask array") return None - def __dask_keys__(self) -> list[Hashable]: + def __dask_keys__(self) -> NestedKeys: if is_duck_dask_array(self._data): return self._data.__dask_keys__() else: From d8007e8aa9c8876d66455e692a85ed6e3a1d1fa7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 17:42:17 +0000 Subject: [PATCH 048/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 27e4be9df47..467fc711d55 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -26,7 +26,6 @@ from dask.typing import ( Graph, NestedKeys, - SchedulerGetCallable, PostComputeCallable, PostPersistCallable, SchedulerGetCallable, From 1f93f5f8d1d8c1d51e52d6a24e0ec220cab336ac Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 19:52:35 +0200 Subject: [PATCH 049/101] Update core.py --- xarray/namedarray/core.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 467fc711d55..ab43096883e 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -31,11 +31,11 @@ SchedulerGetCallable, ) except ImportError: - Graph: typing.Any # typ: ignore[no-redef] - NestedKeys: typing.Any # typ: ignore[no-redef] - SchedulerGetCallable: typing.Any # typ: ignore[no-redef] - PostComputeCallable: typing.Any # typ: ignore[no-redef] - PostPersistCallable: typing.Any # typ: ignore[no-redef] + Graph: typing.Any # type: ignore[no-redef] + NestedKeys: typing.Any # type: ignore[no-redef] + SchedulerGetCallable: typing.Any # type: ignore[no-redef] + PostComputeCallable: typing.Any # type: ignore[no-redef] + PostPersistCallable: typing.Any # type: ignore[no-redef] # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") DimsInput = typing.Union[str, Iterable[Hashable]] @@ -272,9 +272,8 @@ def __dask_tokenize__(self) -> Hashable: # around NetCDF and the like from dask.base import normalize_token - return normalize_token( - (type(self), self._dims, self.data, self.attrs) - ) # type: ignore[no-any-return] + s, d, a, attrs = type(self), self._dims, self.data, self.attrs + return normalize_token((s, d, a, attrs)) # type: ignore[no-any-return] else: raise NotImplementedError("Method requires self.data to be a dask array") From b2570dd58f19e4b759bf3cb3e635b81687b359b9 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:00:58 +0200 Subject: [PATCH 050/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 851cdb94150..27caaae7887 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -1,13 +1,16 @@ from __future__ import annotations -from typing import Any +from typing import Any, TYPE_CHECKING import numpy as np import pytest import xarray as xr from xarray.namedarray.core import NamedArray, as_compatible_data -from xarray.namedarray.utils import Self, T_DuckArray +from xarray.namedarray.utils import T_DuckArray + +if TYPE_CHECKING: + from xarray.namedarray.utils import Self @pytest.fixture From 6835c09f0e3a8aa2adacdbff9e649fc3b15282f2 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:01:06 +0200 Subject: [PATCH 051/101] Update utils.py --- xarray/namedarray/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 9be306ce829..110a36f37b6 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -22,8 +22,8 @@ from dask.array import Array as DaskArray from dask.types import DaskCollection except ImportError: - DaskArray = np.ndarray # type: ignore - DaskCollection: np.Any = np.ndarray # type: ignore + DaskArray = np.ndarray # type: ignore[no-redef] + DaskCollection: np.Any = np.ndarray # type: ignore[no-redef] # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array From 1bac4af26b9f4a1daed29c257a90794c3acbca00 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:03:23 +0200 Subject: [PATCH 052/101] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3125a15802b..e24f88d9679 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ module = [ "cftime.*", "cubed.*", "cupy.*", + "dask.types.*", "fsspec.*", "h5netcdf.*", "h5py.*", From 5d728610c8295971028a8356f64da3dc4456bb6f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 18:04:08 +0000 Subject: [PATCH 053/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_namedarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 27caaae7887..d5b4fb8af32 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np import pytest From ebf47525ce6b30d13b3d7245e22600a30d881de0 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:15:19 +0200 Subject: [PATCH 054/101] Update core.py --- xarray/namedarray/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index ab43096883e..cf533898e3b 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -277,7 +277,7 @@ def __dask_tokenize__(self) -> Hashable: else: raise NotImplementedError("Method requires self.data to be a dask array") - def __dask_graph__(self) -> Graph: + def __dask_graph__(self) -> Graph | None: if is_duck_dask_array(self._data): return self._data.__dask_graph__() else: @@ -298,7 +298,9 @@ def __dask_layers__(self) -> typing.Sequence[str]: raise NotImplementedError("Method requires self.data to be a dask array") @property - def __dask_optimize__(self) -> typing.Callable: + def __dask_optimize__( + self, + ) -> typing.Callable[typing.Any, dict[typing.Any, typing.Any]]: if is_duck_dask_array(self._data): return self._data.__dask_optimize__() else: From 6c8fac963050c963b7e9d4c5c9c86d7dc806e099 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:15:23 +0200 Subject: [PATCH 055/101] Update utils.py --- xarray/namedarray/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 110a36f37b6..6248eca891f 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -22,8 +22,8 @@ from dask.array import Array as DaskArray from dask.types import DaskCollection except ImportError: - DaskArray = np.ndarray # type: ignore[no-redef] - DaskCollection: np.Any = np.ndarray # type: ignore[no-redef] + DaskArray = np.ndarray # type: ignore + DaskCollection: typing.Any = np.ndarray # type: ignore # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array From ee49c5e5d3ed1a6ef5a6e9391732136c33e44807 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:49:33 +0200 Subject: [PATCH 056/101] Update xarray/namedarray/utils.py Co-authored-by: Michael Niklas --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 6248eca891f..f3fde5ddce5 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -61,7 +61,7 @@ def imag(self) -> Self: # Singleton type, as per https://github.com/python/typing/pull/240 class Default(Enum): - token: typing.Literal[0] = 0 + token: typing.Final = 0 _default = Default.token From 5de41420854bdec4fd2938e40a5151a0cabffef2 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:16:14 +0200 Subject: [PATCH 057/101] Update utils.py --- xarray/namedarray/utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 6248eca891f..13e62c48103 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -48,15 +48,26 @@ def real(self) -> Self: def imag(self) -> Self: ... + def astype(self, dtype: np.typing.DTypeLike) -> Self: + ... + # def __array__( # self, dtype: np.typing.DTypeLike = None # ) -> np.ndarray[typing.Any, np.dtype[np.generic]]: # ... +class _ChunkedArray(_Array): + def chunks(self) -> tuple[tuple[int, ...], ...]: + ... + + # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.dtype[np.generic]]) +T_ChunkedArray = typing.TypeVar( + "T_ChunkedArray", bound=_ChunkedArray[np.dtype[np.generic]] +) # Singleton type, as per https://github.com/python/typing/pull/240 @@ -111,6 +122,10 @@ def is_duck_dask_array(x: typing.Any) -> TypeGuard[DaskArray]: return is_duck_array(x) and is_dask_collection(x) +def is_chunked_duck_array(x: T_DuckArray) -> TypeGuard[T_ChunkedArray]: + return hasattr(x, "chunks") + + def to_0d_object_array( value: typing.Any, ) -> np.ndarray[typing.Any, np.dtype[np.object_]]: From 99bf8aa2adbd1574ab5b09bdee6ee6a4f9ccba00 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:16:18 +0200 Subject: [PATCH 058/101] Update core.py --- xarray/namedarray/core.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index cf533898e3b..ee905dbe75a 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -14,6 +14,7 @@ Default, T_DuckArray, _default, + is_chunked_duck_array, is_duck_array, is_duck_dask_array, to_0d_object_array, @@ -188,7 +189,7 @@ def nbytes(self) -> int: the bytes consumed based on the ``size`` and ``dtype``. """ if hasattr(self._data, "nbytes"): - return self._data.nbytes + return self._data.nbytes # type: ignore[no-any-return] else: return self.size * self.dtype.itemsize @@ -298,9 +299,7 @@ def __dask_layers__(self) -> typing.Sequence[str]: raise NotImplementedError("Method requires self.data to be a dask array") @property - def __dask_optimize__( - self, - ) -> typing.Callable[typing.Any, dict[typing.Any, typing.Any]]: + def __dask_optimize__(self) -> typing.Callable[..., dict[typing.Any, typing.Any]]: if is_duck_dask_array(self._data): return self._data.__dask_optimize__() else: @@ -350,7 +349,10 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: NamedArray.chunksizes xarray.unify_chunks """ - return getattr(self._data, "chunks", None) + if is_chunked_duck_array(self._data): + return self._data.chunks + else: + return None @property def chunksizes( @@ -370,7 +372,7 @@ def chunksizes( NamedArray.chunks xarray.unify_chunks """ - if hasattr(self._data, "chunks"): + if is_chunked_duck_array(self._data): return dict(zip(self.dims, self.data.chunks)) else: return {} From b27145ef0a0e525fa8ebeb3f7b00a3d9bff116e0 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:20:16 +0200 Subject: [PATCH 059/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 1bba91e98a2..e181b7450d9 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -57,7 +57,7 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: # ... -class _ChunkedArray(_Array): +class _ChunkedArray(_Array, typing.Protocol[T_DType_co]): def chunks(self) -> tuple[tuple[int, ...], ...]: ... From 7f262d5f3e1433b8fcb723afc2def648d1cd7ba8 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:32:18 +0200 Subject: [PATCH 060/101] Update core.py --- xarray/namedarray/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index ee905dbe75a..68aa6ae7bc4 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -349,8 +349,9 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: NamedArray.chunksizes xarray.unify_chunks """ - if is_chunked_duck_array(self._data): - return self._data.chunks + data = self._data + if is_chunked_duck_array(data): + return data.chunks else: return None @@ -372,8 +373,9 @@ def chunksizes( NamedArray.chunks xarray.unify_chunks """ - if is_chunked_duck_array(self._data): - return dict(zip(self.dims, self.data.chunks)) + data = self._data + if is_chunked_duck_array(data): + return dict(zip(self.dims, data.chunks)) else: return {} From 401a93a9aa5339c999463f5a8a3351047ba9eea6 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:40:10 +0200 Subject: [PATCH 061/101] Update utils.py --- xarray/namedarray/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index e181b7450d9..1eb9b9e4905 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -118,8 +118,8 @@ def is_duck_array(value: typing.Any) -> TypeGuard[T_DuckArray]: ) -def is_duck_dask_array(x: typing.Any) -> TypeGuard[DaskArray]: - return is_duck_array(x) and is_dask_collection(x) +def is_duck_dask_array(x: T_DuckArray) -> TypeGuard[DaskArray]: + return is_dask_collection(x) def is_chunked_duck_array(x: T_DuckArray) -> TypeGuard[T_ChunkedArray]: From bcda5a407db89945f7781f973ac415c752d62285 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 22:22:08 +0200 Subject: [PATCH 062/101] Update core.py --- xarray/namedarray/core.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 68aa6ae7bc4..241567ebb84 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -267,7 +267,7 @@ def imag(self) -> Self: """ return self._replace(data=self.data.imag) - def __dask_tokenize__(self) -> Hashable: + def __dask_tokenize__(self) -> Hashable | None: if is_duck_dask_array(self._data): # Use v.data, instead of v._data, in order to cope with the wrappers # around NetCDF and the like @@ -276,7 +276,7 @@ def __dask_tokenize__(self) -> Hashable: s, d, a, attrs = type(self), self._dims, self.data, self.attrs return normalize_token((s, d, a, attrs)) # type: ignore[no-any-return] else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None def __dask_graph__(self) -> Graph | None: if is_duck_dask_array(self._data): @@ -286,56 +286,58 @@ def __dask_graph__(self) -> Graph | None: # raise NotImplementedError("Method requires self.data to be a dask array") return None - def __dask_keys__(self) -> NestedKeys: + def __dask_keys__(self) -> NestedKeys | None: if is_duck_dask_array(self._data): return self._data.__dask_keys__() else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None - def __dask_layers__(self) -> typing.Sequence[str]: + def __dask_layers__(self) -> typing.Sequence[str] | None: if is_duck_dask_array(self._data): return self._data.__dask_layers__() else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None @property - def __dask_optimize__(self) -> typing.Callable[..., dict[typing.Any, typing.Any]]: + def __dask_optimize__( + self, + ) -> typing.Callable[..., dict[typing.Any, typing.Any]] | None: if is_duck_dask_array(self._data): return self._data.__dask_optimize__() else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None @property - def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable]: + def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable] | None: if is_duck_dask_array(self._data): return self._data.__dask_scheduler__() else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None def __dask_postcompute__( self, - ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]]: + ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]] | None: if is_duck_dask_array(self._data): array_func, array_args = self._data.__dask_postcompute__() return self._dask_finalize, (array_func,) + array_args else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None def __dask_postpersist__( self, - ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]]: + ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]] | None: if is_duck_dask_array(self._data): array_func, array_args = self._data.__dask_postpersist__() return self._dask_finalize, (array_func,) + array_args else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None - def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self: + def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self | None: if is_duck_dask_array(self._data): data = array_func(results, *args, **kwargs) return type(self)(self._dims, data, attrs=self._attrs) else: - raise NotImplementedError("Method requires self.data to be a dask array") + return None @property def chunks(self) -> tuple[tuple[int, ...], ...] | None: @@ -350,7 +352,9 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: xarray.unify_chunks """ data = self._data + # reveal_type(data) if is_chunked_duck_array(data): + # reveal_type(data) return data.chunks else: return None From 2535a5f8f41ca19062acd209cb3a7555fe7190af Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 22:48:38 +0200 Subject: [PATCH 063/101] Update core.py --- xarray/namedarray/core.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 241567ebb84..93aa4d72dbf 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -286,58 +286,59 @@ def __dask_graph__(self) -> Graph | None: # raise NotImplementedError("Method requires self.data to be a dask array") return None - def __dask_keys__(self) -> NestedKeys | None: + def __dask_keys__(self) -> NestedKeys: if is_duck_dask_array(self._data): return self._data.__dask_keys__() else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") - def __dask_layers__(self) -> typing.Sequence[str] | None: + def __dask_layers__(self) -> typing.Sequence[str]: if is_duck_dask_array(self._data): + reveal_type(self._data) return self._data.__dask_layers__() else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") @property def __dask_optimize__( self, - ) -> typing.Callable[..., dict[typing.Any, typing.Any]] | None: + ) -> typing.Callable[..., dict[typing.Any, typing.Any]]: if is_duck_dask_array(self._data): return self._data.__dask_optimize__() else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") @property - def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable] | None: + def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable]: if is_duck_dask_array(self._data): return self._data.__dask_scheduler__() else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") def __dask_postcompute__( self, - ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]] | None: + ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]]: if is_duck_dask_array(self._data): array_func, array_args = self._data.__dask_postcompute__() return self._dask_finalize, (array_func,) + array_args else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") def __dask_postpersist__( self, - ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]] | None: + ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]]: if is_duck_dask_array(self._data): array_func, array_args = self._data.__dask_postpersist__() return self._dask_finalize, (array_func,) + array_args else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") - def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self | None: + def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self: if is_duck_dask_array(self._data): data = array_func(results, *args, **kwargs) return type(self)(self._dims, data, attrs=self._attrs) else: - return None + raise NotImplementedError("Method requires self.data to be a dask array.") @property def chunks(self) -> tuple[tuple[int, ...], ...] | None: From 2c5b49dc1229e9c0770b5097ee89a98cc35efdb5 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 22:56:23 +0200 Subject: [PATCH 064/101] Update core.py --- xarray/namedarray/core.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 93aa4d72dbf..0ffed17422d 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -290,14 +290,13 @@ def __dask_keys__(self) -> NestedKeys: if is_duck_dask_array(self._data): return self._data.__dask_keys__() else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") def __dask_layers__(self) -> typing.Sequence[str]: if is_duck_dask_array(self._data): - reveal_type(self._data) return self._data.__dask_layers__() else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") @property def __dask_optimize__( @@ -306,39 +305,39 @@ def __dask_optimize__( if is_duck_dask_array(self._data): return self._data.__dask_optimize__() else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") @property def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable]: if is_duck_dask_array(self._data): return self._data.__dask_scheduler__() else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") def __dask_postcompute__( self, ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]]: if is_duck_dask_array(self._data): - array_func, array_args = self._data.__dask_postcompute__() + array_func, array_args = self._data.__dask_postcompute__() # type: ignore[no-untyped-call] return self._dask_finalize, (array_func,) + array_args else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") def __dask_postpersist__( self, ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]]: if is_duck_dask_array(self._data): - array_func, array_args = self._data.__dask_postpersist__() + array_func, array_args = self._data.__dask_postpersist__() # type: ignore[no-untyped-call] return self._dask_finalize, (array_func,) + array_args else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self: if is_duck_dask_array(self._data): data = array_func(results, *args, **kwargs) return type(self)(self._dims, data, attrs=self._attrs) else: - raise NotImplementedError("Method requires self.data to be a dask array.") + raise AttributeError("Method requires self.data to be a dask array.") @property def chunks(self) -> tuple[tuple[int, ...], ...] | None: From 9d29827c99f903f9190ba27aff1fb12efd3b0bba Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:06:00 +0200 Subject: [PATCH 065/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index d5b4fb8af32..5638b6b0d83 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -74,7 +74,7 @@ class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): pass array = CustomArray(random_inputs) - output = as_compatible_data(array) + output: T_DuckArray = as_compatible_data(array) assert isinstance(output, np.ndarray) array = CustomArrayIndexable(random_inputs) From 2fba5a9e3b2b378d98118a74e35da002d953b804 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:06:05 +0200 Subject: [PATCH 066/101] Update utils.py --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 1eb9b9e4905..d61278d89d7 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -57,7 +57,7 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: # ... -class _ChunkedArray(_Array, typing.Protocol[T_DType_co]): +class _ChunkedArray(_Array[T_DType_co], typing.Protocol[T_DType_co]): def chunks(self) -> tuple[tuple[int, ...], ...]: ... From 80842eabbb16b7b04f20188a6ff29c208bc3a3d3 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:06:10 +0200 Subject: [PATCH 067/101] Update core.py --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 0ffed17422d..522f8bd92de 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -310,7 +310,7 @@ def __dask_optimize__( @property def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable]: if is_duck_dask_array(self._data): - return self._data.__dask_scheduler__() + return self._data.__dask_scheduler__() # type: ignore[no-any-return] else: raise AttributeError("Method requires self.data to be a dask array.") From cf8d5ccd3d26fa8670a36ee90d5a8d29dfe9f392 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:25:06 +0200 Subject: [PATCH 068/101] Update utils.py --- xarray/namedarray/utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index d61278d89d7..103b5e6440f 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -51,10 +51,8 @@ def imag(self) -> Self: def astype(self, dtype: np.typing.DTypeLike) -> Self: ... - # def __array__( - # self, dtype: np.typing.DTypeLike = None - # ) -> np.ndarray[typing.Any, np.dtype[np.generic]]: - # ... + def __array__(self) -> np.ndarray[typing.Any, T_DType_co]: + ... class _ChunkedArray(_Array[T_DType_co], typing.Protocol[T_DType_co]): @@ -122,7 +120,7 @@ def is_duck_dask_array(x: T_DuckArray) -> TypeGuard[DaskArray]: return is_dask_collection(x) -def is_chunked_duck_array(x: T_DuckArray) -> TypeGuard[T_ChunkedArray]: +def is_chunked_duck_array(x: T_DuckArray | T_ChunkedArray) -> TypeGuard[T_ChunkedArray]: return hasattr(x, "chunks") From 32439fed8cd8cadc77185d2b781fc2cdc442d6ff Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:30:48 +0200 Subject: [PATCH 069/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 5638b6b0d83..f8617b9ef9b 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -70,6 +70,12 @@ def real(self) -> Self: def imag(self) -> Self: raise NotImplementedError + def astype(self, dtype: np.typing.DTypeLike) -> Self: + ... + + def __array__(self) -> np.ndarray[Any, np.dtype[np.generic]]: + ... + class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): pass From 946bd3d0535cc60479a370ccd1b57f56e62649f3 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:31:06 +0200 Subject: [PATCH 070/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index f8617b9ef9b..e8d5f0f552b 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -71,10 +71,10 @@ def imag(self) -> Self: raise NotImplementedError def astype(self, dtype: np.typing.DTypeLike) -> Self: - ... + raise NotImplementedError def __array__(self) -> np.ndarray[Any, np.dtype[np.generic]]: - ... + raise NotImplementedError class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): pass From 130c894cb00dddc166393b570e26bb12add04823 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:53:23 +0200 Subject: [PATCH 071/101] Update core.py --- xarray/namedarray/core.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 522f8bd92de..e06fdf53e6d 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -303,7 +303,7 @@ def __dask_optimize__( self, ) -> typing.Callable[..., dict[typing.Any, typing.Any]]: if is_duck_dask_array(self._data): - return self._data.__dask_optimize__() + return self._data.__dask_optimize__() # type: ignore[no-any-return] else: raise AttributeError("Method requires self.data to be a dask array.") @@ -332,12 +332,15 @@ def __dask_postpersist__( else: raise AttributeError("Method requires self.data to be a dask array.") - def _dask_finalize(self, results, array_func, *args, **kwargs) -> Self: - if is_duck_dask_array(self._data): - data = array_func(results, *args, **kwargs) - return type(self)(self._dims, data, attrs=self._attrs) - else: - raise AttributeError("Method requires self.data to be a dask array.") + def _dask_finalize( + self, + results: T_DuckArray, + array_func: typing.Callable[..., T_DuckArray], + *args: typing.Any, + **kwargs: typing.Any, + ) -> Self: + data = array_func(results, *args, **kwargs) + return type(self)(self._dims, data, attrs=self._attrs) @property def chunks(self) -> tuple[tuple[int, ...], ...] | None: From e0064b988b631d6e6d555df8f559ebc1c6fc74c3 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 00:23:21 +0200 Subject: [PATCH 072/101] Update parallel.py --- xarray/core/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/parallel.py b/xarray/core/parallel.py index 07c3c606bf2..a8ef07a6fd1 100644 --- a/xarray/core/parallel.py +++ b/xarray/core/parallel.py @@ -443,7 +443,7 @@ def subset_dataset_to_block( for dim in variable.dims: chunk = chunk[chunk_index[dim]] - chunk_variable_task = (f"{name}-{gname}-{chunk[0]}",) + chunk_tuple + chunk_variable_task = (f"{name}-{gname}-{chunk[0]:r}",) + chunk_tuple graph[chunk_variable_task] = ( tuple, [variable.dims, chunk, variable.attrs], From fec9f1b547344212d2fdbd6e37db719625e9f581 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 00:23:26 +0200 Subject: [PATCH 073/101] Update utils.py --- xarray/namedarray/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 103b5e6440f..cfcace3d430 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -51,7 +51,9 @@ def imag(self) -> Self: def astype(self, dtype: np.typing.DTypeLike) -> Self: ... - def __array__(self) -> np.ndarray[typing.Any, T_DType_co]: + # TODO: numpy doesn't use any inputs: + # https://github.com/numpy/numpy/blob/v1.24.3/numpy/_typing/_array_like.py#L38 + def __array__(self, *args: typing.Any) -> np.ndarray[typing.Any, T_DType_co]: ... From 877f0f1685aeee7c612fc4bade77270c2be5c38f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 00:38:19 +0200 Subject: [PATCH 074/101] fixes --- xarray/core/parallel.py | 2 +- xarray/namedarray/utils.py | 2 +- xarray/tests/test_namedarray.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/xarray/core/parallel.py b/xarray/core/parallel.py index a8ef07a6fd1..949576b4ee8 100644 --- a/xarray/core/parallel.py +++ b/xarray/core/parallel.py @@ -443,7 +443,7 @@ def subset_dataset_to_block( for dim in variable.dims: chunk = chunk[chunk_index[dim]] - chunk_variable_task = (f"{name}-{gname}-{chunk[0]:r}",) + chunk_tuple + chunk_variable_task = (f"{name}-{gname}-{chunk[0]!r}",) + chunk_tuple graph[chunk_variable_task] = ( tuple, [variable.dims, chunk, variable.attrs], diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index cfcace3d430..641092141a9 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -53,7 +53,7 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: # TODO: numpy doesn't use any inputs: # https://github.com/numpy/numpy/blob/v1.24.3/numpy/_typing/_array_like.py#L38 - def __array__(self, *args: typing.Any) -> np.ndarray[typing.Any, T_DType_co]: + def __array__(self) -> np.ndarray[typing.Any, T_DType_co]: ... diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index e8d5f0f552b..d1be7abb506 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -76,16 +76,16 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: def __array__(self) -> np.ndarray[Any, np.dtype[np.generic]]: raise NotImplementedError - class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): + class CustomArrayIndexable(xr.core.indexing.ExplicitlyIndexed, CustomArray): pass array = CustomArray(random_inputs) - output: T_DuckArray = as_compatible_data(array) + output: CustomArray = as_compatible_data(array) assert isinstance(output, np.ndarray) - array = CustomArrayIndexable(random_inputs) - output = as_compatible_data(array) - assert isinstance(output, CustomArrayIndexable) + array2 = CustomArrayIndexable(random_inputs) + output2: CustomArrayIndexable = as_compatible_data(array2) + assert isinstance(output2, CustomArrayIndexable) def test_properties() -> None: From a5eddb100e64005b678cc935b9a337057b7501ca Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 00:52:00 +0200 Subject: [PATCH 075/101] Update utils.py --- xarray/namedarray/utils.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 641092141a9..5826cc030cf 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -13,11 +13,6 @@ else: from typing_extensions import TypeGuard - if sys.version_info >= (3, 11): - from typing import Self - else: - from typing_extensions import Self - try: from dask.array import Array as DaskArray from dask.types import DaskCollection @@ -26,6 +21,17 @@ DaskCollection: typing.Any = np.ndarray # type: ignore +try: + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self +except ImportError: + if typing.TYPE_CHECKING: + raise + else: + Self: typing.Any = None + # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) # T_DType = typing.TypeVar("T_DType", bound=np.dtype[np.generic]) From 21947153215d04a7b414976e3da0583c760d9c78 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 00:58:18 +0200 Subject: [PATCH 076/101] Update utils.py --- xarray/namedarray/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 5826cc030cf..e069b348b33 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -23,14 +23,16 @@ try: if sys.version_info >= (3, 11): - from typing import Self + from typing import Self as _Self else: - from typing_extensions import Self + from typing_extensions import Self as _Self except ImportError: if typing.TYPE_CHECKING: raise else: - Self: typing.Any = None + _Self: typing.Any = None + +Self = _Self # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) From 77e05f218e4a6e45b9e003d8fd95b6665413ed8f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 01:07:41 +0200 Subject: [PATCH 077/101] ignores --- xarray/namedarray/core.py | 2 +- xarray/namedarray/utils.py | 18 +++++------------- xarray/tests/test_namedarray.py | 2 +- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index e06fdf53e6d..6dfa59a2a2c 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -21,7 +21,7 @@ ) if typing.TYPE_CHECKING: - from xarray.namedarray.utils import Self + from xarray.namedarray.utils import Self # type: ignore[attr-defined] try: from dask.typing import ( diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index e069b348b33..641092141a9 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -13,6 +13,11 @@ else: from typing_extensions import TypeGuard + if sys.version_info >= (3, 11): + from typing import Self + else: + from typing_extensions import Self + try: from dask.array import Array as DaskArray from dask.types import DaskCollection @@ -21,19 +26,6 @@ DaskCollection: typing.Any = np.ndarray # type: ignore -try: - if sys.version_info >= (3, 11): - from typing import Self as _Self - else: - from typing_extensions import Self as _Self -except ImportError: - if typing.TYPE_CHECKING: - raise - else: - _Self: typing.Any = None - -Self = _Self - # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) # T_DType = typing.TypeVar("T_DType", bound=np.dtype[np.generic]) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index d1be7abb506..13a0d4781ec 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -10,7 +10,7 @@ from xarray.namedarray.utils import T_DuckArray if TYPE_CHECKING: - from xarray.namedarray.utils import Self + from xarray.namedarray.utils import Self # type: ignore[attr-defined] @pytest.fixture From 1348df6b480bb996ed875be5458ab23d078624a3 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:55:02 +0200 Subject: [PATCH 078/101] Update xarray/namedarray/utils.py Co-authored-by: Michael Niklas --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 641092141a9..7f06001653a 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -96,7 +96,7 @@ def module_available(module: str) -> bool: return importlib.util.find_spec(module) is not None -def is_dask_collection(x: typing.Any) -> TypeGuard[DaskCollection]: +def is_dask_collection(x: object) -> TypeGuard[DaskCollection]: if module_available("dask"): from dask.typing import DaskCollection From 025e9cc4211b53d3f36929ad03970aaa68c348b1 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:55:26 +0200 Subject: [PATCH 079/101] Update xarray/namedarray/utils.py Co-authored-by: Michael Niklas --- xarray/namedarray/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 7f06001653a..aef1ea56337 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -127,7 +127,7 @@ def is_chunked_duck_array(x: T_DuckArray | T_ChunkedArray) -> TypeGuard[T_Chunke def to_0d_object_array( - value: typing.Any, + value: object, ) -> np.ndarray[typing.Any, np.dtype[np.object_]]: """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" result = np.empty((), dtype=object) From 230521689a7037836a11f536ae87b3ee2697f90a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:56:51 +0200 Subject: [PATCH 080/101] Update core.py --- xarray/namedarray/core.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 6dfa59a2a2c..ffa369fe52d 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -308,9 +308,9 @@ def __dask_optimize__( raise AttributeError("Method requires self.data to be a dask array.") @property - def __dask_scheduler__(self) -> staticmethod[SchedulerGetCallable]: + def __dask_scheduler__(self) -> SchedulerGetCallable: if is_duck_dask_array(self._data): - return self._data.__dask_scheduler__() # type: ignore[no-any-return] + return self._data.__dask_scheduler__ else: raise AttributeError("Method requires self.data to be a dask array.") @@ -325,9 +325,17 @@ def __dask_postcompute__( def __dask_postpersist__( self, - ) -> tuple[PostPersistCallable, tuple[typing.Any, ...]]: + ) -> tuple[ + typing.Callable[ + [T_DuckArray, PostPersistCallable, typing.Any, typing.Any], Self + ], + tuple[typing.Any, ...], + ]: if is_duck_dask_array(self._data): - array_func, array_args = self._data.__dask_postpersist__() # type: ignore[no-untyped-call] + a: tuple[PostPersistCallable, tuple[typing.Any, ...]] + a = self._data.__dask_postpersist__() # type: ignore[no-untyped-call] + array_func, array_args = a + return self._dask_finalize, (array_func,) + array_args else: raise AttributeError("Method requires self.data to be a dask array.") @@ -335,7 +343,7 @@ def __dask_postpersist__( def _dask_finalize( self, results: T_DuckArray, - array_func: typing.Callable[..., T_DuckArray], + array_func: PostPersistCallable, *args: typing.Any, **kwargs: typing.Any, ) -> Self: @@ -355,10 +363,12 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: xarray.unify_chunks """ data = self._data - # reveal_type(data) + reveal_type(data) # note: Revealed type is "T_DuckArray`1" if is_chunked_duck_array(data): - # reveal_type(data) - return data.chunks + reveal_type(data) # note: Revealed type is "" + return ( + data.chunks + ) # error: has no attribute "chunks" [attr-defined] else: return None From 13c8953ac435642ae6335f355521afb60e99546a Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:56:56 +0200 Subject: [PATCH 081/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 13a0d4781ec..0e986c66621 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -56,11 +56,11 @@ def __init__(self, array: T_DuckArray) -> None: @property def dtype(self) -> np.dtype[np.generic]: - raise NotImplementedError + raise self.array.dtype @property def shape(self) -> tuple[int, ...]: - raise NotImplementedError + return self.array.shape @property def real(self) -> Self: From 5559548444ebfa77396e19ae8872c9c359810a2f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:21:48 +0200 Subject: [PATCH 082/101] Update core.py --- xarray/namedarray/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index ffa369fe52d..1aa2408a8c0 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -327,12 +327,13 @@ def __dask_postpersist__( self, ) -> tuple[ typing.Callable[ - [T_DuckArray, PostPersistCallable, typing.Any, typing.Any], Self + [T_DuckArray, PostPersistCallable[..., typing.Any], typing.Any, typing.Any], + Self, ], tuple[typing.Any, ...], ]: if is_duck_dask_array(self._data): - a: tuple[PostPersistCallable, tuple[typing.Any, ...]] + a: tuple[PostPersistCallable[..., typing.Any], tuple[typing.Any, ...]] a = self._data.__dask_postpersist__() # type: ignore[no-untyped-call] array_func, array_args = a From a177ce7f82da4c85a1eb9a3821a4a25fa7f7bb9f Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:25:44 +0200 Subject: [PATCH 083/101] Update core.py --- xarray/namedarray/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 1aa2408a8c0..a5c24e02268 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -327,13 +327,13 @@ def __dask_postpersist__( self, ) -> tuple[ typing.Callable[ - [T_DuckArray, PostPersistCallable[..., typing.Any], typing.Any, typing.Any], + [T_DuckArray, PostPersistCallable[typing.Any], typing.Any, typing.Any], Self, ], tuple[typing.Any, ...], ]: if is_duck_dask_array(self._data): - a: tuple[PostPersistCallable[..., typing.Any], tuple[typing.Any, ...]] + a: tuple[PostPersistCallable[typing.Any], tuple[typing.Any, ...]] a = self._data.__dask_postpersist__() # type: ignore[no-untyped-call] array_func, array_args = a From ca7ee3782434bd92dc403751ca38850ffd4cceff Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:41:34 +0200 Subject: [PATCH 084/101] Update core.py --- xarray/namedarray/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index a5c24e02268..e3142c73f61 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -327,7 +327,7 @@ def __dask_postpersist__( self, ) -> tuple[ typing.Callable[ - [T_DuckArray, PostPersistCallable[typing.Any], typing.Any, typing.Any], + [Graph, PostPersistCallable[typing.Any], typing.Any, typing.Any], Self, ], tuple[typing.Any, ...], @@ -343,7 +343,7 @@ def __dask_postpersist__( def _dask_finalize( self, - results: T_DuckArray, + results: Graph, array_func: PostPersistCallable, *args: typing.Any, **kwargs: typing.Any, From ce77930cb918e68c407cf99505adb66017a2194d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:45:53 +0200 Subject: [PATCH 085/101] Update core.py --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index e3142c73f61..e9d49708316 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -344,7 +344,7 @@ def __dask_postpersist__( def _dask_finalize( self, results: Graph, - array_func: PostPersistCallable, + array_func: PostPersistCallable[typing.Any], *args: typing.Any, **kwargs: typing.Any, ) -> Self: From 99f6c9b4ce26dae74a8d72475c0179950dd3d4f1 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:52:21 +0200 Subject: [PATCH 086/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 0e986c66621..c2106937a26 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -56,7 +56,7 @@ def __init__(self, array: T_DuckArray) -> None: @property def dtype(self) -> np.dtype[np.generic]: - raise self.array.dtype + return self.array.dtype @property def shape(self) -> tuple[int, ...]: @@ -76,7 +76,7 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: def __array__(self) -> np.ndarray[Any, np.dtype[np.generic]]: raise NotImplementedError - class CustomArrayIndexable(xr.core.indexing.ExplicitlyIndexed, CustomArray): + class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): pass array = CustomArray(random_inputs) From 0fa4fd3461f2e4bbe81c7659349c5da99c9e273b Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:53:49 +0200 Subject: [PATCH 087/101] Update core.py --- xarray/namedarray/core.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index e9d49708316..62529e753f3 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -364,12 +364,8 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: xarray.unify_chunks """ data = self._data - reveal_type(data) # note: Revealed type is "T_DuckArray`1" if is_chunked_duck_array(data): - reveal_type(data) # note: Revealed type is "" - return ( - data.chunks - ) # error: has no attribute "chunks" [attr-defined] + return data.chunks else: return None From 5b98dd5ed1c81f4eccde55c3238e48c355c2ede1 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:00:12 +0200 Subject: [PATCH 088/101] Update test_namedarray.py --- xarray/tests/test_namedarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index c2106937a26..0fd901ce974 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -74,7 +74,7 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: raise NotImplementedError def __array__(self) -> np.ndarray[Any, np.dtype[np.generic]]: - raise NotImplementedError + return np.array(self.array) class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): pass From 56a77555348b659c024ec241b17c771b7e0bb419 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:13:48 +0200 Subject: [PATCH 089/101] import specific type functions --- xarray/namedarray/core.py | 66 +++++++++++++++++++------------------- xarray/namedarray/utils.py | 28 ++++++++-------- 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 62529e753f3..ae5d23d50af 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -2,7 +2,7 @@ import copy import math -import typing +from typing import TYPE_CHECKING, Any, Union, cast, Generic, Sequence, Callable from collections.abc import Hashable, Iterable, Mapping import numpy as np @@ -20,7 +20,7 @@ to_0d_object_array, ) -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from xarray.namedarray.utils import Self # type: ignore[attr-defined] try: @@ -32,16 +32,16 @@ SchedulerGetCallable, ) except ImportError: - Graph: typing.Any # type: ignore[no-redef] - NestedKeys: typing.Any # type: ignore[no-redef] - SchedulerGetCallable: typing.Any # type: ignore[no-redef] - PostComputeCallable: typing.Any # type: ignore[no-redef] - PostPersistCallable: typing.Any # type: ignore[no-redef] + Graph: Any # type: ignore[no-redef] + NestedKeys: Any # type: ignore[no-redef] + SchedulerGetCallable: Any # type: ignore[no-redef] + PostComputeCallable: Any # type: ignore[no-redef] + PostPersistCallable: Any # type: ignore[no-redef] # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") - DimsInput = typing.Union[str, Iterable[Hashable]] + DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] - AttrsInput = typing.Union[Mapping[typing.Any, typing.Any], None] + AttrsInput = Union[Mapping[Any, Any], None] # TODO: Add tests! @@ -50,7 +50,7 @@ def as_compatible_data( ) -> T_DuckArray: if fastpath and getattr(data, "ndim", 0) > 0: # can't use fastpath (yet) for scalars - return typing.cast(T_DuckArray, data) + return cast(T_DuckArray, data) if isinstance(data, np.ma.MaskedArray): mask = np.ma.getmaskarray(data) # type: ignore[no-untyped-call] @@ -58,24 +58,24 @@ def as_compatible_data( # TODO: requires refactoring/vendoring xarray.core.dtypes and xarray.core.duck_array_ops raise NotImplementedError("MaskedArray is not supported yet") else: - return typing.cast(T_DuckArray, np.asarray(data)) + return cast(T_DuckArray, np.asarray(data)) if is_duck_array(data): return data if isinstance(data, NamedArray): - return typing.cast(T_DuckArray, data.data) + return cast(T_DuckArray, data.data) if isinstance(data, ExplicitlyIndexed): # TODO: better that is_duck_array(ExplicitlyIndexed) -> True - return typing.cast(T_DuckArray, data) + return cast(T_DuckArray, data) if isinstance(data, tuple): data = to_0d_object_array(data) # validate whether the data is valid data types. - return typing.cast(T_DuckArray, np.asarray(data)) + return cast(T_DuckArray, np.asarray(data)) -class NamedArray(typing.Generic[T_DuckArray]): +class NamedArray(Generic[T_DuckArray]): """A lightweight wrapper around duck arrays with named dimensions and attributes which describe a single Array. Numeric operations on this object implement array broadcasting and dimension alignment based on dimension names, @@ -85,7 +85,7 @@ class NamedArray(typing.Generic[T_DuckArray]): _data: T_DuckArray _dims: Dims - _attrs: dict[typing.Any, typing.Any] | None + _attrs: dict[Any, Any] | None def __init__( self, @@ -151,7 +151,7 @@ def __len__(self) -> int: raise TypeError("len() of unsized object") from exc @property - def dtype(self) -> np.dtype[typing.Any]: + def dtype(self) -> np.dtype[Any]: """ Data-type of the array’s elements. @@ -212,14 +212,14 @@ def _parse_dimensions(self, dims: DimsInput) -> Dims: return dims @property - def attrs(self) -> dict[typing.Any, typing.Any]: + def attrs(self) -> dict[Any, Any]: """Dictionary of local attributes on this NamedArray.""" if self._attrs is None: self._attrs = {} return self._attrs @attrs.setter - def attrs(self, value: Mapping[typing.Any, typing.Any]) -> None: + def attrs(self, value: Mapping[Any, Any]) -> None: self._attrs = dict(value) def _check_shape(self, new_data: T_DuckArray) -> None: @@ -292,7 +292,7 @@ def __dask_keys__(self) -> NestedKeys: else: raise AttributeError("Method requires self.data to be a dask array.") - def __dask_layers__(self) -> typing.Sequence[str]: + def __dask_layers__(self) -> Sequence[str]: if is_duck_dask_array(self._data): return self._data.__dask_layers__() else: @@ -301,7 +301,7 @@ def __dask_layers__(self) -> typing.Sequence[str]: @property def __dask_optimize__( self, - ) -> typing.Callable[..., dict[typing.Any, typing.Any]]: + ) -> Callable[..., dict[Any, Any]]: if is_duck_dask_array(self._data): return self._data.__dask_optimize__() # type: ignore[no-any-return] else: @@ -316,7 +316,7 @@ def __dask_scheduler__(self) -> SchedulerGetCallable: def __dask_postcompute__( self, - ) -> tuple[PostComputeCallable, tuple[typing.Any, ...]]: + ) -> tuple[PostComputeCallable, tuple[Any, ...]]: if is_duck_dask_array(self._data): array_func, array_args = self._data.__dask_postcompute__() # type: ignore[no-untyped-call] return self._dask_finalize, (array_func,) + array_args @@ -326,14 +326,14 @@ def __dask_postcompute__( def __dask_postpersist__( self, ) -> tuple[ - typing.Callable[ - [Graph, PostPersistCallable[typing.Any], typing.Any, typing.Any], + Callable[ + [Graph, PostPersistCallable[Any], Any, Any], Self, ], - tuple[typing.Any, ...], + tuple[Any, ...], ]: if is_duck_dask_array(self._data): - a: tuple[PostPersistCallable[typing.Any], tuple[typing.Any, ...]] + a: tuple[PostPersistCallable[Any], tuple[Any, ...]] a = self._data.__dask_postpersist__() # type: ignore[no-untyped-call] array_func, array_args = a @@ -344,9 +344,9 @@ def __dask_postpersist__( def _dask_finalize( self, results: Graph, - array_func: PostPersistCallable[typing.Any], - *args: typing.Any, - **kwargs: typing.Any, + array_func: PostPersistCallable[Any], + *args: Any, + **kwargs: Any, ) -> Self: data = array_func(results, *args, **kwargs) return type(self)(self._dims, data, attrs=self._attrs) @@ -372,7 +372,7 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: @property def chunksizes( self, - ) -> typing.Mapping[typing.Any, tuple[int, ...]]: + ) -> Mapping[Any, tuple[int, ...]]: """ Mapping from dimension names to block lengths for this namedArray's data, or None if the underlying data is not a dask array. @@ -416,7 +416,7 @@ def _copy( self, deep: bool = True, data: T_DuckArray | np.typing.ArrayLike | None = None, - memo: dict[int, typing.Any] | None = None, + memo: dict[int, Any] | None = None, ) -> Self: if data is None: ndata = self._data @@ -435,7 +435,7 @@ def _copy( def __copy__(self) -> Self: return self._copy(deep=False) - def __deepcopy__(self, memo: dict[int, typing.Any] | None = None) -> Self: + def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: return self._copy(deep=True, memo=memo) def copy( @@ -480,7 +480,7 @@ def _nonzero(self) -> tuple[Self, ...]: def _as_sparse( self, sparse_format: str | Default = _default, - fill_value: typing.Any = dtypes.NA, + fill_value: object = dtypes.NA, ) -> Self: """ use sparse-array as backend. diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index aef1ea56337..c4cb082f8be 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -2,12 +2,12 @@ import importlib import sys -import typing +from typing import TYPE_CHECKING, Any, TypeVar, Protocol, Final from enum import Enum import numpy as np -if typing.TYPE_CHECKING: +if TYPE_CHECKING: if sys.version_info >= (3, 10): from typing import TypeGuard else: @@ -23,15 +23,15 @@ from dask.types import DaskCollection except ImportError: DaskArray = np.ndarray # type: ignore - DaskCollection: typing.Any = np.ndarray # type: ignore + DaskCollection: Any = np.ndarray # type: ignore # https://stackoverflow.com/questions/74633074/how-to-type-hint-a-generic-numpy-array -T_DType_co = typing.TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) -# T_DType = typing.TypeVar("T_DType", bound=np.dtype[np.generic]) +T_DType_co = TypeVar("T_DType_co", bound=np.dtype[np.generic], covariant=True) +# T_DType = TypeVar("T_DType", bound=np.dtype[np.generic]) -class _Array(typing.Protocol[T_DType_co]): +class _Array(Protocol[T_DType_co]): @property def dtype(self) -> T_DType_co: ... @@ -53,26 +53,24 @@ def astype(self, dtype: np.typing.DTypeLike) -> Self: # TODO: numpy doesn't use any inputs: # https://github.com/numpy/numpy/blob/v1.24.3/numpy/_typing/_array_like.py#L38 - def __array__(self) -> np.ndarray[typing.Any, T_DType_co]: + def __array__(self) -> np.ndarray[Any, T_DType_co]: ... -class _ChunkedArray(_Array[T_DType_co], typing.Protocol[T_DType_co]): +class _ChunkedArray(_Array[T_DType_co], Protocol[T_DType_co]): def chunks(self) -> tuple[tuple[int, ...], ...]: ... # temporary placeholder for indicating an array api compliant type. # hopefully in the future we can narrow this down more -T_DuckArray = typing.TypeVar("T_DuckArray", bound=_Array[np.dtype[np.generic]]) -T_ChunkedArray = typing.TypeVar( - "T_ChunkedArray", bound=_ChunkedArray[np.dtype[np.generic]] -) +T_DuckArray = TypeVar("T_DuckArray", bound=_Array[np.dtype[np.generic]]) +T_ChunkedArray = TypeVar("T_ChunkedArray", bound=_ChunkedArray[np.dtype[np.generic]]) # Singleton type, as per https://github.com/python/typing/pull/240 class Default(Enum): - token: typing.Final = 0 + token: Final = 0 _default = Default.token @@ -104,7 +102,7 @@ def is_dask_collection(x: object) -> TypeGuard[DaskCollection]: return False -def is_duck_array(value: typing.Any) -> TypeGuard[T_DuckArray]: +def is_duck_array(value: object) -> TypeGuard[T_DuckArray]: if isinstance(value, np.ndarray): return True return ( @@ -128,7 +126,7 @@ def is_chunked_duck_array(x: T_DuckArray | T_ChunkedArray) -> TypeGuard[T_Chunke def to_0d_object_array( value: object, -) -> np.ndarray[typing.Any, np.dtype[np.object_]]: +) -> np.ndarray[Any, np.dtype[np.object_]]: """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" result = np.empty((), dtype=object) result[()] = value From b321a8470aeb1262bd874cb081a1eb878832b13d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:17:06 +0000 Subject: [PATCH 090/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 4 ++-- xarray/namedarray/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index ae5d23d50af..b2535f692f9 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -2,8 +2,8 @@ import copy import math -from typing import TYPE_CHECKING, Any, Union, cast, Generic, Sequence, Callable -from collections.abc import Hashable, Iterable, Mapping +from collections.abc import Hashable, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Callable, Generic, Union, cast import numpy as np diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index c4cb082f8be..11ca1811a42 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -2,8 +2,8 @@ import importlib import sys -from typing import TYPE_CHECKING, Any, TypeVar, Protocol, Final from enum import Enum +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar import numpy as np From 6a33331c1e2dae4b7bb831bf2763583fc25c5063 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:44:50 +0200 Subject: [PATCH 091/101] Update core.py --- xarray/namedarray/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index b2535f692f9..f0fca5947cd 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -38,7 +38,7 @@ PostComputeCallable: Any # type: ignore[no-redef] PostPersistCallable: Any # type: ignore[no-redef] - # T_NamedArray = typing.TypeVar("T_NamedArray", bound="NamedArray") + # T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray") DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] AttrsInput = Union[Mapping[Any, Any], None] @@ -480,7 +480,7 @@ def _nonzero(self) -> tuple[Self, ...]: def _as_sparse( self, sparse_format: str | Default = _default, - fill_value: object = dtypes.NA, + fill_value: np.typing.ArrayLike | Default = _default, ) -> Self: """ use sparse-array as backend. @@ -488,7 +488,7 @@ def _as_sparse( import sparse # TODO: what to do if dask-backended? - if fill_value is dtypes.NA: + if fill_value is _default: dtype, fill_value = dtypes.maybe_promote(self.dtype) else: dtype = dtypes.result_type(self.dtype, fill_value) From 863ed1d1c3ec8b883b8141bd001b6a81df365ec4 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:23:25 +0200 Subject: [PATCH 092/101] Update core.py --- xarray/namedarray/core.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index f0fca5947cd..c1da3603d78 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -268,15 +268,12 @@ def imag(self) -> Self: return self._replace(data=self.data.imag) def __dask_tokenize__(self) -> Hashable | None: - if is_duck_dask_array(self._data): - # Use v.data, instead of v._data, in order to cope with the wrappers - # around NetCDF and the like - from dask.base import normalize_token + # Use v.data, instead of v._data, in order to cope with the wrappers + # around NetCDF and the like + from dask.base import normalize_token - s, d, a, attrs = type(self), self._dims, self.data, self.attrs - return normalize_token((s, d, a, attrs)) # type: ignore[no-any-return] - else: - return None + s, d, a, attrs = type(self), self._dims, self.data, self.attrs + return normalize_token((s, d, a, attrs)) # type: ignore[no-any-return] def __dask_graph__(self) -> Graph | None: if is_duck_dask_array(self._data): From 11b36faa832b8ee75a963565146d9cab9bdc0323 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:39:15 +0200 Subject: [PATCH 093/101] Update core.py --- xarray/namedarray/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index c1da3603d78..8be011218b6 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -12,6 +12,7 @@ from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.utils import ( Default, + T_ChunkedArray, T_DuckArray, _default, is_chunked_duck_array, @@ -300,7 +301,7 @@ def __dask_optimize__( self, ) -> Callable[..., dict[Any, Any]]: if is_duck_dask_array(self._data): - return self._data.__dask_optimize__() # type: ignore[no-any-return] + return self._data.__dask_optimize__ # type: ignore[no-any-return] else: raise AttributeError("Method requires self.data to be a dask array.") @@ -360,8 +361,10 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: NamedArray.chunksizes xarray.unify_chunks """ - data = self._data + data: T_DuckArray | T_ChunkedArray = self._data + # reveal_type(data) if is_chunked_duck_array(data): + # reveal_type(data) return data.chunks else: return None From 2bd6f8cdcac79e892531baff98417e406f35a8f5 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:53:14 +0200 Subject: [PATCH 094/101] Try chunkedarray instead --- xarray/namedarray/core.py | 4 +--- xarray/namedarray/utils.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 8be011218b6..07d40c5063b 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -361,10 +361,8 @@ def chunks(self) -> tuple[tuple[int, ...], ...] | None: NamedArray.chunksizes xarray.unify_chunks """ - data: T_DuckArray | T_ChunkedArray = self._data - # reveal_type(data) + data = self._data if is_chunked_duck_array(data): - # reveal_type(data) return data.chunks else: return None diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 11ca1811a42..4f8aa544986 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -58,6 +58,7 @@ def __array__(self) -> np.ndarray[Any, T_DType_co]: class _ChunkedArray(_Array[T_DType_co], Protocol[T_DType_co]): + @property def chunks(self) -> tuple[tuple[int, ...], ...]: ... @@ -120,7 +121,7 @@ def is_duck_dask_array(x: T_DuckArray) -> TypeGuard[DaskArray]: return is_dask_collection(x) -def is_chunked_duck_array(x: T_DuckArray | T_ChunkedArray) -> TypeGuard[T_ChunkedArray]: +def is_chunked_duck_array(x: T_DuckArray) -> TypeGuard[_ChunkedArray]: return hasattr(x, "chunks") From 476dda25eb6dc752daa1cc9005ae9fc46106d140 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:53:54 +0000 Subject: [PATCH 095/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 07d40c5063b..938aecc0b61 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -12,7 +12,6 @@ from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.utils import ( Default, - T_ChunkedArray, T_DuckArray, _default, is_chunked_duck_array, From 27c18b8d73e20eb3eab24fb3bdaa2ec09ba1c0a9 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:07:29 +0200 Subject: [PATCH 096/101] fixes --- xarray/namedarray/core.py | 8 ++++---- xarray/namedarray/utils.py | 4 +++- xarray/tests/test_namedarray.py | 5 +++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 07d40c5063b..82516f5709a 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -3,7 +3,7 @@ import copy import math from collections.abc import Hashable, Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Callable, Generic, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, Union, cast import numpy as np @@ -39,7 +39,7 @@ PostComputeCallable: Any # type: ignore[no-redef] PostPersistCallable: Any # type: ignore[no-redef] - # T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray") + T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray") DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] AttrsInput = Union[Mapping[Any, Any], None] @@ -479,7 +479,7 @@ def _as_sparse( self, sparse_format: str | Default = _default, fill_value: np.typing.ArrayLike | Default = _default, - ) -> Self: + ) -> T_NamedArray: """ use sparse-array as backend. """ @@ -501,7 +501,7 @@ def _as_sparse( data = as_sparse(self.data.astype(dtype), fill_value=fill_value) return self._replace(data=data) - def _to_dense(self) -> Self: + def _to_dense(self) -> T_NamedArray | Self: """ Change backend from sparse to np.array """ diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 4f8aa544986..c77009aeb2d 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -121,7 +121,9 @@ def is_duck_dask_array(x: T_DuckArray) -> TypeGuard[DaskArray]: return is_dask_collection(x) -def is_chunked_duck_array(x: T_DuckArray) -> TypeGuard[_ChunkedArray]: +def is_chunked_duck_array( + x: T_DuckArray, +) -> TypeGuard[_ChunkedArray[np.dtype[np.generic]]]: return hasattr(x, "chunks") diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 0fd901ce974..ea1588bf554 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -50,7 +50,7 @@ def test_as_compatible_data_with_explicitly_indexed( random_inputs: np.ndarray[Any, Any] ) -> None: # TODO: Make xr.core.indexing.ExplicitlyIndexed pass is_duck_array and remove this test. - class CustomArray(xr.core.indexing.NDArrayMixin): + class CustomArrayBase(xr.core.indexing.NDArrayMixin): def __init__(self, array: T_DuckArray) -> None: self.array = array @@ -73,10 +73,11 @@ def imag(self) -> Self: def astype(self, dtype: np.typing.DTypeLike) -> Self: raise NotImplementedError + class CustomArray(CustomArrayBase): def __array__(self) -> np.ndarray[Any, np.dtype[np.generic]]: return np.array(self.array) - class CustomArrayIndexable(CustomArray, xr.core.indexing.ExplicitlyIndexed): + class CustomArrayIndexable(CustomArrayBase, xr.core.indexing.ExplicitlyIndexed): pass array = CustomArray(random_inputs) From c2a1fb7e1511fb29128ed486c8659c0e10107897 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:19:43 +0200 Subject: [PATCH 097/101] Update core.py --- xarray/namedarray/core.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index bb66bd48593..4f762499cdf 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -38,6 +38,11 @@ PostComputeCallable: Any # type: ignore[no-redef] PostPersistCallable: Any # type: ignore[no-redef] + try: + from sparse import SparseArray + except ImportError: + SparseArray: Any # type: ignore[no-redef] + T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray") DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] @@ -478,7 +483,7 @@ def _as_sparse( self, sparse_format: str | Default = _default, fill_value: np.typing.ArrayLike | Default = _default, - ) -> T_NamedArray: + ) -> T_NamedArray[SparseArray]: """ use sparse-array as backend. """ @@ -500,7 +505,7 @@ def _as_sparse( data = as_sparse(self.data.astype(dtype), fill_value=fill_value) return self._replace(data=data) - def _to_dense(self) -> T_NamedArray | Self: + def _to_dense(self) -> T_NamedArray[T_DuckArray] | Self: """ Change backend from sparse to np.array """ From 892e83d36a7872cf561f442802adefb9450b7e07 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:24:48 +0200 Subject: [PATCH 098/101] Update core.py --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 4f762499cdf..5f0ba563cfd 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -43,7 +43,7 @@ except ImportError: SparseArray: Any # type: ignore[no-redef] - T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray") + T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray[T_DuckArray]") DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] AttrsInput = Union[Mapping[Any, Any], None] From 58266c4f6546117e25073d345766f7e7b28572ad Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:29:35 +0200 Subject: [PATCH 099/101] Update core.py --- xarray/namedarray/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 5f0ba563cfd..94f38f9f826 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -43,7 +43,7 @@ except ImportError: SparseArray: Any # type: ignore[no-redef] - T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray[T_DuckArray]") + # T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray[T_DuckArray]") DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...] AttrsInput = Union[Mapping[Any, Any], None] @@ -483,7 +483,7 @@ def _as_sparse( self, sparse_format: str | Default = _default, fill_value: np.typing.ArrayLike | Default = _default, - ) -> T_NamedArray[SparseArray]: + ) -> Self: """ use sparse-array as backend. """ @@ -505,7 +505,7 @@ def _as_sparse( data = as_sparse(self.data.astype(dtype), fill_value=fill_value) return self._replace(data=data) - def _to_dense(self) -> T_NamedArray[T_DuckArray] | Self: + def _to_dense(self) -> Self: """ Change backend from sparse to np.array """ From 00cba3db6e06a3577768e9092cb81e90e2461c70 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:30:16 +0000 Subject: [PATCH 100/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 94f38f9f826..6bff1146be6 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -3,7 +3,7 @@ import copy import math from collections.abc import Hashable, Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Generic, Union, cast import numpy as np From 114c45c8dd1a85b54f8b3f131016d79ad3fa752e Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:33:12 +0200 Subject: [PATCH 101/101] Update core.py --- xarray/namedarray/core.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 94f38f9f826..15b87d04201 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -38,11 +38,6 @@ PostComputeCallable: Any # type: ignore[no-redef] PostPersistCallable: Any # type: ignore[no-redef] - try: - from sparse import SparseArray - except ImportError: - SparseArray: Any # type: ignore[no-redef] - # T_NamedArray = TypeVar("T_NamedArray", bound="NamedArray[T_DuckArray]") DimsInput = Union[str, Iterable[Hashable]] Dims = tuple[Hashable, ...]