From 02c296905f9e12299eccbe3267049c2ce6ccf933 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 22 Oct 2019 18:30:43 +0000 Subject: [PATCH 1/9] Add cftime git tip to upstream-dev + temporarily pin cftime (#3431) * Add cftime git tip to upstream-dev + temporarily pin cftime * specify cftime min version as 1.0.0 * Min cftime version 1.0.3 * wn --- ci/azure/install.yml | 3 ++- ci/requirements/py36-min-all-deps.yml | 2 +- ci/requirements/py36.yml | 2 +- ci/requirements/py37-windows.yml | 2 +- ci/requirements/py37.yml | 2 +- doc/whats-new.rst | 5 +++++ 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 8d72ffce124..f1192f4424f 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -24,7 +24,8 @@ steps: --upgrade \ git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ - git+https://github.com/zarr-developers/zarr + git+https://github.com/zarr-developers/zarr \ + git+https://github.com/Unidata/cftime condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 3f10a158f91..bbc51d09ce2 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -13,7 +13,7 @@ dependencies: - cartopy=0.17 - cdms2=3.1 - cfgrib=0.9 - - cftime=1.0 + - cftime=1.0.3 - coveralls - dask=1.2 - distributed=1.27 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 4d6d778f884..54ab9b5be7a 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime + - cftime=1.0.3.4 - coveralls - dask - distributed diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 78784b48f3c..3318d837257 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -9,7 +9,7 @@ dependencies: - cartopy # - cdms2 # Not available on Windows # - cfgrib>=0.9.2 # Causes Python interpreter crash on Windows - - cftime + - cftime=1.0.3.4 - coveralls - dask - distributed diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 38e5db641b7..5cdb634649c 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime + - cftime=1.0.3.4 - coveralls - dask - distributed diff --git a/doc/whats-new.rst b/doc/whats-new.rst index abe472cc6bb..ede18c3b147 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -18,6 +18,11 @@ What's New v0.14.1 (unreleased) -------------------- +Breaking changes +~~~~~~~~~~~~~~~~ + +- Minimum cftime version is now 1.0.3. By `Deepak Cherian `_. + New Features ~~~~~~~~~~~~ - Added integration tests against `pint `_. From e258b88ae9229217095e55ea451d3c9546d8fe6e Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe Date: Tue, 22 Oct 2019 12:31:34 -0600 Subject: [PATCH 2/9] Sync with latest version of cftime (v1.0.4) (#3430) * Remove `dayofwk=-1`-- not needed for cftime>=1.0.4 * Maintain backwards compatibility Co-Authored-By: Spencer Clark * Maintain backwards compatibility Co-Authored-By: Spencer Clark * Maintain backwards compatibility * Add missing import * Update whats-new --- doc/whats-new.rst | 4 ++++ xarray/coding/cftime_offsets.py | 16 +++++++++++----- xarray/coding/cftimeindex.py | 14 ++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index ede18c3b147..2fbfd58ddba 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,10 @@ Bug fixes - Fix regression introduced in v0.14.0 that would cause a crash if dask is installed but cloudpickle isn't (:issue:`3401`) by `Rhys Doyle `_ +- Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4. + By `Anderson Banihirwe `_. + + Documentation ~~~~~~~~~~~~~ diff --git a/xarray/coding/cftime_offsets.py b/xarray/coding/cftime_offsets.py index af46510f7c4..8471ed1a558 100644 --- a/xarray/coding/cftime_offsets.py +++ b/xarray/coding/cftime_offsets.py @@ -50,6 +50,7 @@ from ..core.pdcompat import count_not_none from .cftimeindex import CFTimeIndex, _parse_iso8601_with_reso from .times import format_cftime_datetime +from distutils.version import LooseVersion def get_date_type(calendar): @@ -222,6 +223,8 @@ def _adjust_n_years(other, n, month, reference_day): def _shift_month(date, months, day_option="start"): """Shift the date to a month start or end a given number of months away. """ + import cftime + delta_year = (date.month + months) // 12 month = (date.month + months) % 12 @@ -237,11 +240,14 @@ def _shift_month(date, months, day_option="start"): day = _days_in_month(reference) else: raise ValueError(day_option) - # dayofwk=-1 is required to update the dayofwk and dayofyr attributes of - # the returned date object in versions of cftime between 1.0.2 and - # 1.0.3.4. It can be removed for versions of cftime greater than - # 1.0.3.4. - return date.replace(year=year, month=month, day=day, dayofwk=-1) + if LooseVersion(cftime.__version__) < LooseVersion("1.0.4"): + # dayofwk=-1 is required to update the dayofwk and dayofyr attributes of + # the returned date object in versions of cftime between 1.0.2 and + # 1.0.3.4. It can be removed for versions of cftime greater than + # 1.0.3.4. + return date.replace(year=year, month=month, day=day, dayofwk=-1) + else: + return date.replace(year=year, month=month, day=day) def roll_qtrday(other, n, month, day_option, modby=3): diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 434d55d6569..559c5e16287 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -96,6 +96,8 @@ def parse_iso8601(datetime_string): def _parse_iso8601_with_reso(date_type, timestr): + import cftime + default = date_type(1, 1, 1) result = parse_iso8601(timestr) replace = {} @@ -107,12 +109,12 @@ def _parse_iso8601_with_reso(date_type, timestr): # TODO: Consider adding support for sub-second resolution? replace[attr] = int(value) resolution = attr - - # dayofwk=-1 is required to update the dayofwk and dayofyr attributes of - # the returned date object in versions of cftime between 1.0.2 and - # 1.0.3.4. It can be removed for versions of cftime greater than - # 1.0.3.4. - replace["dayofwk"] = -1 + if LooseVersion(cftime.__version__) < LooseVersion("1.0.4"): + # dayofwk=-1 is required to update the dayofwk and dayofyr attributes of + # the returned date object in versions of cftime between 1.0.2 and + # 1.0.3.4. It can be removed for versions of cftime greater than + # 1.0.3.4. + replace["dayofwk"] = -1 return default.replace(**replace), resolution From a3e43e6f1f5827cb635b48ba69ec4c1ac312d811 Mon Sep 17 00:00:00 2001 From: Huite Date: Tue, 22 Oct 2019 20:45:23 +0200 Subject: [PATCH 3/9] Avoid multiplication DeprecationWarning in rasterio backend (#3428) --- xarray/backends/rasterio_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 7430633c5c0..1f9b9943573 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -257,8 +257,8 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, loc if parse: nx, ny = riods.width, riods.height # xarray coordinates are pixel centered - x, _ = (np.arange(nx) + 0.5, np.zeros(nx) + 0.5) * riods.transform - _, y = (np.zeros(ny) + 0.5, np.arange(ny) + 0.5) * riods.transform + x, _ = riods.transform * (np.arange(nx) + 0.5, np.zeros(nx) + 0.5) + _, y = riods.transform * (np.zeros(ny) + 0.5, np.arange(ny) + 0.5) coords["y"] = y coords["x"] = x else: From 72be873857a0a56659c905ce491ca1f94b44fd5c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 22 Oct 2019 19:42:08 +0000 Subject: [PATCH 4/9] Test that Dataset and DataArray resampling are identical (#3412) --- xarray/tests/test_dataset.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index dce417f27f9..006d6881b5a 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -3593,6 +3593,19 @@ def test_resample_old_api(self): with raises_regex(TypeError, r"resample\(\) no longer supports"): ds.resample("1D", dim="time") + def test_resample_ds_da_are_the_same(self): + time = pd.date_range("2000-01-01", freq="6H", periods=365 * 4) + ds = xr.Dataset( + { + "foo": (("time", "x"), np.random.randn(365 * 4, 5)), + "time": time, + "x": np.arange(5), + } + ) + assert_identical( + ds.resample(time="M").mean()["foo"], ds.foo.resample(time="M").mean() + ) + def test_ds_resample_apply_func_args(self): def func(arg1, arg2, arg3=0.0): return arg1.mean("time") + arg2 + arg3 From c8dac5866d2c54ee6b262b5060a701e0be1e40cb Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Oct 2019 02:06:09 +0100 Subject: [PATCH 5/9] MAGA (Make Azure Green Again) (#3436) * Support for patch version in min_deps_check.py * Fix requirements * Fix requirements * cftime is 1.0.4 incompatible * Use cftime PR#127 * trivial * trivial --- ci/azure/install.yml | 5 +- ci/min_deps_check.py | 72 ++++++++++++++++----------- ci/requirements/py36-min-all-deps.yml | 2 +- ci/requirements/py36.yml | 4 +- ci/requirements/py37-windows.yml | 6 +-- ci/requirements/py37.yml | 4 +- doc/whats-new.rst | 2 +- 7 files changed, 55 insertions(+), 40 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index f1192f4424f..2911e227172 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -15,17 +15,18 @@ steps: --no-deps \ --pre \ --upgrade \ - numpy \ matplotlib \ pandas \ scipy + # numpy \ # FIXME https://github.com/pydata/xarray/issues/3409 pip install \ --no-deps \ --upgrade \ git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ - git+https://github.com/Unidata/cftime + git+https://github.com/Unidata/cftime.git@refs/pull/127/merge + # git+https://github.com/Unidata/cftime # FIXME PR 127 not merged yet condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 3bdd48ca76d..a5ba90679b7 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -6,7 +6,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta -from typing import Dict, Iterator, Tuple +from typing import Dict, Iterator, Optional, Tuple import yaml @@ -34,10 +34,14 @@ def error(msg: str) -> None: print("ERROR:", msg) -def parse_requirements(fname) -> Iterator[Tuple[str, int, int]]: +def warning(msg: str) -> None: + print("WARNING:", msg) + + +def parse_requirements(fname) -> Iterator[Tuple[str, int, int, Optional[int]]]: """Load requirements/py36-min-all-deps.yml - Yield (package name, major version, minor version) + Yield (package name, major version, minor version, [patch version]) """ global has_errors @@ -52,15 +56,18 @@ def parse_requirements(fname) -> Iterator[Tuple[str, int, int]]: if pkg.endswith("<") or pkg.endswith(">") or eq != "=": error("package should be pinned with exact version: " + row) continue + try: - major, minor = version.split(".") - except ValueError: - error("expected major.minor (without patch): " + row) - continue - try: - yield pkg, int(major), int(minor) + version_tup = tuple(int(x) for x in version.split(".")) except ValueError: - error("failed to parse version: " + row) + raise ValueError("non-numerical version: " + row) + + if len(version_tup) == 2: + yield (pkg, *version_tup, None) # type: ignore + elif len(version_tup) == 3: + yield (pkg, *version_tup) # type: ignore + else: + raise ValueError("expected major.minor or major.minor.patch: " + row) def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: @@ -80,9 +87,9 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: label = label.strip() if label == "file name": value = value.strip()[len(pkg) :] - major, minor = value.split("-")[1].split(".")[:2] - major = int(major) - minor = int(minor) + smajor, sminor = value.split("-")[1].split(".")[:2] + major = int(smajor) + minor = int(sminor) if label == "timestamp": assert major is not None assert minor is not None @@ -109,17 +116,15 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: def process_pkg( - pkg: str, req_major: int, req_minor: int -) -> Tuple[str, int, int, str, int, int, str, str]: + pkg: str, req_major: int, req_minor: int, req_patch: Optional[int] +) -> Tuple[str, str, str, str, str, str]: """Compare package version from requirements file to available versions in conda. Return row to build pandas dataframe: - package name - - major version in requirements file - - minor version in requirements file + - major.minor.[patch] version in requirements file - publication date of version in requirements file (YYYY-MM-DD) - - major version suggested by policy - - minor version suggested by policy + - major.minor version suggested by policy - publication date of version suggested by policy (YYYY-MM-DD) - status ("<", "=", "> (!)") """ @@ -130,7 +135,7 @@ def process_pkg( req_published = versions[req_major, req_minor] except KeyError: error("not found in conda: " + pkg) - return pkg, req_major, req_minor, "-", 0, 0, "-", "(!)" + return pkg, fmt_version(req_major, req_minor, req_patch), "-", "-", "-", "(!)" policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT) policy_published = datetime.now() - timedelta(days=policy_months * 30) @@ -153,30 +158,39 @@ def process_pkg( else: status = "=" + if req_patch is not None: + warning("patch version should not appear in requirements file: " + pkg) + status += " (w)" + return ( pkg, - req_major, - req_minor, + fmt_version(req_major, req_minor, req_patch), req_published.strftime("%Y-%m-%d"), - policy_major, - policy_minor, + fmt_version(policy_major, policy_minor), policy_published_actual.strftime("%Y-%m-%d"), status, ) +def fmt_version(major: int, minor: int, patch: int = None) -> str: + if patch is None: + return f"{major}.{minor}" + else: + return f"{major}.{minor}.{patch}" + + def main() -> None: fname = sys.argv[1] with ThreadPoolExecutor(8) as ex: futures = [ - ex.submit(process_pkg, pkg, major, minor) - for pkg, major, minor in parse_requirements(fname) + ex.submit(process_pkg, pkg, major, minor, patch) + for pkg, major, minor, patch in parse_requirements(fname) ] rows = [f.result() for f in futures] - print("Package Required Policy Status") - print("------------- ----------------- ----------------- ------") - fmt = "{:13} {:>1d}.{:<2d} ({:10}) {:>1d}.{:<2d} ({:10}) {}" + print("Package Required Policy Status") + print("------------- -------------------- -------------------- ------") + fmt = "{:13} {:7} ({:10}) {:7} ({:10}) {}" for row in rows: print(fmt.format(*row)) diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index bbc51d09ce2..c99ae39e5d9 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -13,7 +13,7 @@ dependencies: - cartopy=0.17 - cdms2=3.1 - cfgrib=0.9 - - cftime=1.0.3 + - cftime=1.0.3 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask=1.2 - distributed=1.27 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 54ab9b5be7a..6e27cea2ffe 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime=1.0.3.4 + - cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy + - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 - pandas - pint - pip diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 3318d837257..7027fc11ab7 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -8,8 +8,8 @@ dependencies: - bottleneck - cartopy # - cdms2 # Not available on Windows - # - cfgrib>=0.9.2 # Causes Python interpreter crash on Windows - - cftime=1.0.3.4 + # - cfgrib # Causes Python interpreter crash on Windows + - cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy + - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 - pandas - pint - pip diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 5cdb634649c..a4c974c0176 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -9,7 +9,7 @@ dependencies: - cartopy - cdms2 - cfgrib - - cftime=1.0.3.4 + - cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken - coveralls - dask - distributed @@ -25,7 +25,7 @@ dependencies: - nc-time-axis - netcdf4 - numba - - numpy + - numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409 - pandas - pint - pip diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 2fbfd58ddba..0f4d0c10f1f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,7 +42,7 @@ Bug fixes but cloudpickle isn't (:issue:`3401`) by `Rhys Doyle `_ - Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4. - By `Anderson Banihirwe `_. + By `Anderson Banihirwe `_. Documentation From 9c6b457bd022ee4beba395bb06e52a766dc18078 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 23 Oct 2019 18:28:20 -0700 Subject: [PATCH 6/9] Use cftime master for upstream-dev build (#3439) --- ci/azure/install.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 2911e227172..fee886ba804 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -25,8 +25,7 @@ steps: git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ git+https://github.com/zarr-developers/zarr \ - git+https://github.com/Unidata/cftime.git@refs/pull/127/merge - # git+https://github.com/Unidata/cftime # FIXME PR 127 not merged yet + git+https://github.com/Unidata/cftime condition: eq(variables['UPSTREAM_DEV'], 'true') displayName: Install upstream dev dependencies From 35c75f557a510b7e81c70fcaad8cad419a12ee2e Mon Sep 17 00:00:00 2001 From: Jon Thielen Date: Wed, 23 Oct 2019 23:25:42 -0500 Subject: [PATCH 7/9] Update Terminology page to account for multidimensional coordinates (#3410) * Update Terminology page to account for multidimensional coordinates * Update xarray-docs conda environment file * Add change to whats-new.rst * Modify example description for multidimensional coords based on suggestion --- doc/contributing.rst | 4 ++-- doc/terminology.rst | 6 +++--- doc/whats-new.rst | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 66e8377600e..028ec47e014 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -286,12 +286,12 @@ How to build the *xarray* documentation Requirements ~~~~~~~~~~~~ Make sure to follow the instructions on :ref:`creating a development environment above `, but -to build the docs you need to use the environment file ``doc/environment.yml``. +to build the docs you need to use the environment file ``ci/requirements/doc.yml``. .. code-block:: none # Create and activate the docs environment - conda env create -f doc/environment.yml + conda env create -f ci/requirements/doc.yml conda activate xarray-docs # or with older versions of Anaconda: diff --git a/doc/terminology.rst b/doc/terminology.rst index 138a99740fe..4ee56190d5f 100644 --- a/doc/terminology.rst +++ b/doc/terminology.rst @@ -27,15 +27,15 @@ Terminology ---- -**Coordinate:** An array that labels a dimension of another ``DataArray``. Loosely, the coordinate array's values can be thought of as tick labels along a dimension. There are two types of coordinate arrays: *dimension coordinates* and *non-dimension coordinates* (see below). A coordinate named ``x`` can be retrieved from ``arr.coords[x]``. A ``DataArray`` can have more coordinates than dimensions because a single dimension can be assigned multiple coordinate arrays. However, only one coordinate array can be a assigned as a particular dimension's dimension coordinate array. As a consequence, ``len(arr.dims) <= len(arr.coords)`` in general. +**Coordinate:** An array that labels a dimension or set of dimensions of another ``DataArray``. In the usual one-dimensional case, the coordinate array's values can loosely be thought of as tick labels along a dimension. There are two types of coordinate arrays: *dimension coordinates* and *non-dimension coordinates* (see below). A coordinate named ``x`` can be retrieved from ``arr.coords[x]``. A ``DataArray`` can have more coordinates than dimensions because a single dimension can be labeled by multiple coordinate arrays. However, only one coordinate array can be a assigned as a particular dimension's dimension coordinate array. As a consequence, ``len(arr.dims) <= len(arr.coords)`` in general. ---- -**Dimension coordinate:** A coordinate array assigned to ``arr`` with both a name and dimension name in ``arr.dims``. Dimension coordinates are used for label-based indexing and alignment, like the index found on a :py:class:`pandas.DataFrame` or :py:class:`pandas.Series`. In fact, dimension coordinates use :py:class:`pandas.Index` objects under the hood for efficient computation. Dimension coordinates are marked by ``*`` when printing a ``DataArray`` or ``Dataset``. +**Dimension coordinate:** A one-dimensional coordinate array assigned to ``arr`` with both a name and dimension name in ``arr.dims``. Dimension coordinates are used for label-based indexing and alignment, like the index found on a :py:class:`pandas.DataFrame` or :py:class:`pandas.Series`. In fact, dimension coordinates use :py:class:`pandas.Index` objects under the hood for efficient computation. Dimension coordinates are marked by ``*`` when printing a ``DataArray`` or ``Dataset``. ---- -**Non-dimension coordinate:** A coordinate array assigned to ``arr`` with a name in ``arr.dims`` but a dimension name *not* in ``arr.dims``. These coordinate arrays are useful for auxiliary labeling. However, non-dimension coordinates are not indexed, and any operation on non-dimension coordinates that leverages indexing will fail. Printing ``arr.coords`` will print all of ``arr``'s coordinate names, with the assigned dimensions in parentheses. For example, ``coord_name (dim_name) 1 2 3 ...``. +**Non-dimension coordinate:** A coordinate array assigned to ``arr`` with a name in ``arr.coords`` but *not* in ``arr.dims``. These coordinates arrays can be one-dimensional or multidimensional, and they are useful for auxiliary labeling. As an example, multidimensional coordinates are often used in geoscience datasets when :doc:`the data's physical coordinates (such as latitude and longitude) differ from their logical coordinates `. However, non-dimension coordinates are not indexed, and any operation on non-dimension coordinates that leverages indexing will fail. Printing ``arr.coords`` will print all of ``arr``'s coordinate names, with the corresponding dimension(s) in parentheses. For example, ``coord_name (dim_name) 1 2 3 ...``. ---- diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0f4d0c10f1f..9d3e64badb8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -52,6 +52,8 @@ Documentation :py:meth:`Dataset.resample` and explicitly state that a datetime-like dimension is required. (:pull:`3400`) By `Justus Magin `_. +- Update the terminology page to address multidimensional coordinates. (:pull:`3410`) + By `Jon Thielen `_. Internal Changes ~~~~~~~~~~~~~~~~ From 52e4ef1a79ecf51691887e3c4bf3994be9a231ee Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 24 Oct 2019 12:43:35 +0100 Subject: [PATCH 8/9] Hack around #3440 (#3442) --- ci/azure/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index fee886ba804..8da0ac1b5de 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -16,7 +16,7 @@ steps: --pre \ --upgrade \ matplotlib \ - pandas \ + pandas=0.26.0.dev0+628.g03c1a3db2 \ # FIXME https://github.com/pydata/xarray/issues/3440 scipy # numpy \ # FIXME https://github.com/pydata/xarray/issues/3409 pip install \ From 652dd3ca77dd19bbd1ab21fe556340c1904ec382 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 24 Oct 2019 13:53:20 +0100 Subject: [PATCH 9/9] minor lint tweaks (#3429) * pyflakes 2.1.1 and f-strings * Tweaks from mypy 0.740 (will skip to 0.750: mypy#7735) * black -t py36 * isort * fix tests --- xarray/backends/h5netcdf_.py | 2 +- xarray/backends/zarr.py | 2 +- xarray/core/accessor_dt.py | 4 ++- xarray/core/arithmetic.py | 2 +- xarray/core/coordinates.py | 2 +- xarray/core/dataset.py | 10 +++---- xarray/core/indexing.py | 51 +++++++++++++--------------------- xarray/core/missing.py | 8 +++--- xarray/core/resample.py | 2 +- xarray/core/rolling.py | 5 ++-- xarray/plot/__init__.py | 2 +- xarray/plot/facetgrid.py | 8 +++--- xarray/tests/test_dataarray.py | 6 ++-- xarray/tutorial.py | 2 +- 14 files changed, 47 insertions(+), 59 deletions(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 92c29502994..51ed512f98b 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -245,7 +245,7 @@ def prepare_variable( dtype=dtype, dimensions=variable.dims, fillvalue=fillvalue, - **kwargs + **kwargs, ) else: nc4_var = self.ds[name] diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index d924e1da4fc..6d4ebb02a11 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -467,7 +467,7 @@ def open_zarr( drop_variables=None, consolidated=False, overwrite_encoded_chunks=False, - **kwargs + **kwargs, ): """Load and decode a dataset from a Zarr store. diff --git a/xarray/core/accessor_dt.py b/xarray/core/accessor_dt.py index 832eb88c5fa..aff6fbc6691 100644 --- a/xarray/core/accessor_dt.py +++ b/xarray/core/accessor_dt.py @@ -178,7 +178,9 @@ def __init__(self, obj): ) self._obj = obj - def _tslib_field_accessor(name, docstring=None, dtype=None): + def _tslib_field_accessor( # type: ignore + name: str, docstring: str = None, dtype: np.dtype = None + ): def f(self, dtype=dtype): if dtype is None: dtype = self._obj.dtype diff --git a/xarray/core/arithmetic.py b/xarray/core/arithmetic.py index 137db034c95..571dfbe70ed 100644 --- a/xarray/core/arithmetic.py +++ b/xarray/core/arithmetic.py @@ -76,7 +76,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): dataset_join=dataset_join, dataset_fill_value=np.nan, kwargs=kwargs, - dask="allowed" + dask="allowed", ) # this has no runtime function - these are listed so IDEs know these diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index 0c11e8efa38..eb2ceb1be07 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -367,7 +367,7 @@ def remap_label_indexers( indexers: Mapping[Hashable, Any] = None, method: str = None, tolerance=None, - **indexers_kwargs: Any + **indexers_kwargs: Any, ) -> Tuple[dict, dict]: # TODO more precise return type after annotations in indexing """Remap indexers from obj.coords. If indexer is an instance of DataArray and it has coordinate, then this coordinate diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 3fdde8fa4e3..12d5cbdc9f3 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -2967,15 +2967,13 @@ def expand_dims( for a in axis: if a < -result_ndim or result_ndim - 1 < a: raise IndexError( - "Axis {a} is out of bounds of the expanded" - " dimension size {dim}.".format( - a=a, v=k, dim=result_ndim - ) + f"Axis {a} of variable {k} is out of bounds of the " + f"expanded dimension size {result_ndim}" ) axis_pos = [a if a >= 0 else result_ndim + a for a in axis] if len(axis_pos) != len(set(axis_pos)): - raise ValueError("axis should not contain duplicate" " values.") + raise ValueError("axis should not contain duplicate values") # We need to sort them to make sure `axis` equals to the # axis positions of the result array. zip_axis_dim = sorted(zip(axis_pos, dim.items())) @@ -3131,7 +3129,7 @@ def reorder_levels( coord = self._variables[dim] index = self.indexes[dim] if not isinstance(index, pd.MultiIndex): - raise ValueError("coordinate %r has no MultiIndex" % dim) + raise ValueError(f"coordinate {dim} has no MultiIndex") new_index = index.reorder_levels(order) variables[dim] = IndexVariable(coord.dims, new_index) indexes[dim] = new_index diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 3d2e634eaa8..b9809a8d2b9 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -59,8 +59,7 @@ def _sanitize_slice_element(x): if isinstance(x, np.ndarray): if x.ndim != 0: raise ValueError( - "cannot use non-scalar arrays in a slice for " - "xarray indexing: {}".format(x) + f"cannot use non-scalar arrays in a slice for xarray indexing: {x}" ) x = x[()] @@ -128,9 +127,9 @@ def convert_label_indexer(index, label, index_name="", method=None, tolerance=No # unlike pandas, in xarray we never want to silently convert a # slice indexer into an array indexer raise KeyError( - "cannot represent labeled-based slice indexer for " - "dimension %r with a slice over integer positions; " - "the index is unsorted or non-unique" % index_name + "cannot represent labeled-based slice indexer for dimension " + f"{index_name!r} with a slice over integer positions; the index is " + "unsorted or non-unique" ) elif is_dict_like(label): @@ -190,7 +189,7 @@ def convert_label_indexer(index, label, index_name="", method=None, tolerance=No ) indexer = get_indexer_nd(index, label, method, tolerance) if np.any(indexer < 0): - raise KeyError("not all values found in index %r" % index_name) + raise KeyError(f"not all values found in index {index_name!r}") return indexer, new_index @@ -208,7 +207,7 @@ def get_dim_indexers(data_obj, indexers): if k not in data_obj.dims and k not in data_obj._level_coords ] if invalid: - raise ValueError("dimensions or multi-index levels %r do not exist" % invalid) + raise ValueError(f"dimensions or multi-index levels {invalid!r} do not exist") level_indexers = defaultdict(dict) dim_indexers = {} @@ -223,8 +222,8 @@ def get_dim_indexers(data_obj, indexers): for dim, level_labels in level_indexers.items(): if dim_indexers.get(dim, False): raise ValueError( - "cannot combine multi-index level indexers " - "with an indexer for dimension %s" % dim + "cannot combine multi-index level indexers with an indexer for " + f"dimension {dim}" ) dim_indexers[dim] = level_labels @@ -326,7 +325,7 @@ def tuple(self): return self._key def __repr__(self): - return "{}({})".format(type(self).__name__, self.tuple) + return f"{type(self).__name__}({self.tuple})" def as_integer_or_none(value): @@ -362,9 +361,7 @@ def __init__(self, key): k = as_integer_slice(k) else: raise TypeError( - "unexpected indexer type for {}: {!r}".format( - type(self).__name__, k - ) + f"unexpected indexer type for {type(self).__name__}: {k!r}" ) new_key.append(k) @@ -395,20 +392,17 @@ def __init__(self, key): elif isinstance(k, np.ndarray): if not np.issubdtype(k.dtype, np.integer): raise TypeError( - "invalid indexer array, does not have " - "integer dtype: {!r}".format(k) + f"invalid indexer array, does not have integer dtype: {k!r}" ) if k.ndim != 1: raise TypeError( - "invalid indexer array for {}, must have " - "exactly 1 dimension: ".format(type(self).__name__, k) + f"invalid indexer array for {type(self).__name__}; must have " + f"exactly 1 dimension: {k!r}" ) k = np.asarray(k, dtype=np.int64) else: raise TypeError( - "unexpected indexer type for {}: {!r}".format( - type(self).__name__, k - ) + f"unexpected indexer type for {type(self).__name__}: {k!r}" ) new_key.append(k) @@ -439,8 +433,7 @@ def __init__(self, key): elif isinstance(k, np.ndarray): if not np.issubdtype(k.dtype, np.integer): raise TypeError( - "invalid indexer array, does not have " - "integer dtype: {!r}".format(k) + f"invalid indexer array, does not have integer dtype: {k!r}" ) if ndim is None: ndim = k.ndim @@ -448,14 +441,12 @@ def __init__(self, key): ndims = [k.ndim for k in key if isinstance(k, np.ndarray)] raise ValueError( "invalid indexer key: ndarray arguments " - "have different numbers of dimensions: {}".format(ndims) + f"have different numbers of dimensions: {ndims}" ) k = np.asarray(k, dtype=np.int64) else: raise TypeError( - "unexpected indexer type for {}: {!r}".format( - type(self).__name__, k - ) + f"unexpected indexer type for {type(self).__name__}: {k!r}" ) new_key.append(k) @@ -574,9 +565,7 @@ def __setitem__(self, key, value): self.array[full_key] = value def __repr__(self): - return "{}(array={!r}, key={!r})".format( - type(self).__name__, self.array, self.key - ) + return f"{type(self).__name__}(array={self.array!r}, key={self.key!r})" class LazilyVectorizedIndexedArray(ExplicitlyIndexedNDArrayMixin): @@ -627,9 +616,7 @@ def __setitem__(self, key, value): ) def __repr__(self): - return "{}(array={!r}, key={!r})".format( - type(self).__name__, self.array, self.key - ) + return f"{type(self).__name__}(array={self.array!r}, key={self.key!r})" def _wrap_numpy_scalars(array): diff --git a/xarray/core/missing.py b/xarray/core/missing.py index dfe209e3f7e..77dde66484e 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -71,7 +71,7 @@ def __call__(self, x): self._yi, left=self._left, right=self._right, - **self.call_kwargs + **self.call_kwargs, ) @@ -93,7 +93,7 @@ def __init__( copy=False, bounds_error=False, order=None, - **kwargs + **kwargs, ): from scipy.interpolate import interp1d @@ -126,7 +126,7 @@ def __init__( bounds_error=False, assume_sorted=assume_sorted, copy=copy, - **self.cons_kwargs + **self.cons_kwargs, ) @@ -147,7 +147,7 @@ def __init__( order=3, nu=0, ext=None, - **kwargs + **kwargs, ): from scipy.interpolate import UnivariateSpline diff --git a/xarray/core/resample.py b/xarray/core/resample.py index 1f2e5c0be43..998964273be 100644 --- a/xarray/core/resample.py +++ b/xarray/core/resample.py @@ -151,7 +151,7 @@ def _interpolate(self, kind="linear"): assume_sorted=True, method=kind, kwargs={"bounds_error": False}, - **{self._dim: self._full_index} + **{self._dim: self._full_index}, ) diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index d22c6aa7d91..f4e571a8efe 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -1,4 +1,5 @@ import functools +from typing import Callable import numpy as np @@ -106,7 +107,7 @@ def __repr__(self): def __len__(self): return self.obj.sizes[self.dim] - def _reduce_method(name): + def _reduce_method(name: str) -> Callable: # type: ignore array_agg_func = getattr(duck_array_ops, name) bottleneck_move_func = getattr(bottleneck, "move_" + name, None) @@ -453,7 +454,7 @@ def _numpy_or_bottleneck_reduce( array_agg_func=array_agg_func, bottleneck_move_func=bottleneck_move_func, ), - **kwargs + **kwargs, ) def construct(self, window_dim, stride=1, fill_value=dtypes.NA): diff --git a/xarray/plot/__init__.py b/xarray/plot/__init__.py index 903321228f7..86a09506824 100644 --- a/xarray/plot/__init__.py +++ b/xarray/plot/__init__.py @@ -1,6 +1,6 @@ +from .dataset_plot import scatter from .facetgrid import FacetGrid from .plot import contour, contourf, hist, imshow, line, pcolormesh, plot, step -from .dataset_plot import scatter __all__ = [ "plot", diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index ec51ff26c07..7f13ba601fe 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -294,7 +294,7 @@ def map_dataarray_line( hue=hue, add_legend=False, _labels=False, - **kwargs + **kwargs, ) self._mappables.append(mappable) @@ -376,7 +376,7 @@ def add_legend(self, **kwargs): labels=list(self._hue_var.values), title=self._hue_label, loc="center right", - **kwargs + **kwargs, ) self.figlegend = figlegend @@ -491,7 +491,7 @@ def set_titles(self, template="{coord} = {value}", maxchar=30, size=None, **kwar rotation=270, ha="left", va="center", - **kwargs + **kwargs, ) # The column titles on the top row @@ -590,7 +590,7 @@ def _easy_facetgrid( subplot_kws=None, ax=None, figsize=None, - **kwargs + **kwargs, ): """ Convenience method to call xarray.plot.FacetGrid from 2d plotting methods diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index d05a02ae705..a3a2f55f6cc 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -1534,11 +1534,11 @@ def test_expand_dims_error(self): # Should not pass the already existing dimension. array.expand_dims(dim=["x"]) # raise if duplicate - with raises_regex(ValueError, "duplicate values."): + with raises_regex(ValueError, "duplicate values"): array.expand_dims(dim=["y", "y"]) - with raises_regex(ValueError, "duplicate values."): + with raises_regex(ValueError, "duplicate values"): array.expand_dims(dim=["y", "z"], axis=[1, 1]) - with raises_regex(ValueError, "duplicate values."): + with raises_regex(ValueError, "duplicate values"): array.expand_dims(dim=["y", "z"], axis=[2, -2]) # out of bounds error, axis must be in [-4, 3] diff --git a/xarray/tutorial.py b/xarray/tutorial.py index 88ca8d3ab4f..e99c0632fe8 100644 --- a/xarray/tutorial.py +++ b/xarray/tutorial.py @@ -32,7 +32,7 @@ def open_dataset( cache_dir=_default_cache_dir, github_url="https://github.com/pydata/xarray-data", branch="master", - **kws + **kws, ): """ Open a dataset from the online repository (requires internet).