ensure warnings cannot become errors in assert_

doc/whats-new.rst

@@ -126,6 +126,8 @@ Internal Changes
   in ipython (:issue:`4741`, :pull:`4742`). By `Richard Kleijn <https://github.com/rhkleijn>`_.
 - Added the ``set_close`` method to ``Dataset`` and ``DataArray`` for beckends to specify how to voluntary release
   all resources. (:pull:`#4809`), By `Alessandro Amici <https://github.com/alexamici>`_.
+- Ensure warnings cannot be turned into exceptions in :py:func:`testing.assert_equal` and
+  the other `assert_*` functions (:pull:`4864`). By `Mathias Hauser <https://github.com/mathause>`_.
 
 .. _whats-new.0.16.2:
 
@@ -142,7 +144,7 @@ Deprecations
 
 - :py:attr:`~core.accessor_dt.DatetimeAccessor.weekofyear` and :py:attr:`~core.accessor_dt.DatetimeAccessor.week`
   have been deprecated. Use ``DataArray.dt.isocalendar().week``
-  instead (:pull:`4534`). By `Mathias Hauser <https://github.com/mathause>`_,
+  instead (:pull:`4534`). By `Mathias Hauser <https://github.com/mathause>`_.
   `Maximilian Roos <https://github.com/max-sixty>`_, and `Spencer Clark <https://github.com/spencerkclark>`_.
 - :py:attr:`DataArray.rolling` and :py:attr:`Dataset.rolling` no longer support passing ``keep_attrs``
   via its constructor. Pass ``keep_attrs`` via the applied function, i.e. use

xarray/testing.py

@@ -1,5 +1,6 @@
 """Testing functions exposed to the user API"""
 import functools
+import warnings
 from typing import Hashable, Set, Union
 
 import numpy as np
@@ -60,13 +61,18 @@ def assert_equal(a, b):
     numpy.testing.assert_array_equal
     """
     __tracebackhide__ = True
-    assert type(a) == type(b)
-    if isinstance(a, (Variable, DataArray)):
-        assert a.equals(b), formatting.diff_array_repr(a, b, "equals")
-    elif isinstance(a, Dataset):
-        assert a.equals(b), formatting.diff_dataset_repr(a, b, "equals")
-    else:
-        raise TypeError("{} not supported by assertion comparison".format(type(a)))
+
+    # sometimes tests elevate warnings to errors -> make sure that does not happen here
+    with warnings.catch_warnings():
+        warnings.simplefilter("always")
+
+        assert type(a) == type(b)
+        if isinstance(a, (Variable, DataArray)):
+            assert a.equals(b), formatting.diff_array_repr(a, b, "equals")
+        elif isinstance(a, Dataset):
+            assert a.equals(b), formatting.diff_dataset_repr(a, b, "equals")
+        else:
+            raise TypeError("{} not supported by assertion comparison".format(type(a)))
 
 
 def assert_identical(a, b):
@@ -87,16 +93,21 @@ def assert_identical(a, b):
     assert_equal, assert_allclose, Dataset.equals, DataArray.equals
     """
     __tracebackhide__ = True
-    assert type(a) == type(b)
-    if isinstance(a, Variable):
-        assert a.identical(b), formatting.diff_array_repr(a, b, "identical")
-    elif isinstance(a, DataArray):
-        assert a.name == b.name
-        assert a.identical(b), formatting.diff_array_repr(a, b, "identical")
-    elif isinstance(a, (Dataset, Variable)):
-        assert a.identical(b), formatting.diff_dataset_repr(a, b, "identical")
-    else:
-        raise TypeError("{} not supported by assertion comparison".format(type(a)))
+
+    # sometimes tests elevate warnings to errors -> make sure that does not happen here
+    with warnings.catch_warnings():
+        warnings.simplefilter("always")
+
+        assert type(a) == type(b)
+        if isinstance(a, Variable):
+            assert a.identical(b), formatting.diff_array_repr(a, b, "identical")
+        elif isinstance(a, DataArray):
+            assert a.name == b.name
+            assert a.identical(b), formatting.diff_array_repr(a, b, "identical")
+        elif isinstance(a, (Dataset, Variable)):
+            assert a.identical(b), formatting.diff_dataset_repr(a, b, "identical")
+        else:
+            raise TypeError("{} not supported by assertion comparison".format(type(a)))
 
 
 def assert_allclose(a, b, rtol=1e-05, atol=1e-08, decode_bytes=True):
@@ -138,21 +149,25 @@ def compat_variable(a, b):
 
         return a.dims == b.dims and (a._data is b._data or equiv(a.data, b.data))
 
-    if isinstance(a, Variable):
-        allclose = compat_variable(a, b)
-        assert allclose, formatting.diff_array_repr(a, b, compat=equiv)
-    elif isinstance(a, DataArray):
-        allclose = utils.dict_equiv(
-            a.coords, b.coords, compat=compat_variable
-        ) and compat_variable(a.variable, b.variable)
-        assert allclose, formatting.diff_array_repr(a, b, compat=equiv)
-    elif isinstance(a, Dataset):
-        allclose = a._coord_names == b._coord_names and utils.dict_equiv(
-            a.variables, b.variables, compat=compat_variable
-        )
-        assert allclose, formatting.diff_dataset_repr(a, b, compat=equiv)
-    else:
-        raise TypeError("{} not supported by assertion comparison".format(type(a)))
+    # sometimes tests elevate warnings to errors -> make sure that does not happen here
+    with warnings.catch_warnings():
+        warnings.simplefilter("always")
+
+        if isinstance(a, Variable):
+            allclose = compat_variable(a, b)
+            assert allclose, formatting.diff_array_repr(a, b, compat=equiv)
+        elif isinstance(a, DataArray):
+            allclose = utils.dict_equiv(
+                a.coords, b.coords, compat=compat_variable
+            ) and compat_variable(a.variable, b.variable)
+            assert allclose, formatting.diff_array_repr(a, b, compat=equiv)
+        elif isinstance(a, Dataset):
+            allclose = a._coord_names == b._coord_names and utils.dict_equiv(
+                a.variables, b.variables, compat=compat_variable
+            )
+            assert allclose, formatting.diff_dataset_repr(a, b, compat=equiv)
+        else:
+            raise TypeError("{} not supported by assertion comparison".format(type(a)))
 
 
 def _format_message(x, y, err_msg, verbose):
@@ -188,8 +203,15 @@ def assert_duckarray_allclose(
     """ Like `np.testing.assert_allclose`, but for duckarrays. """
     __tracebackhide__ = True
 
-    allclose = duck_array_ops.allclose_or_equiv(actual, desired, rtol=rtol, atol=atol)
-    assert allclose, _format_message(actual, desired, err_msg=err_msg, verbose=verbose)
+    with warnings.catch_warnings():
+        warnings.simplefilter("always")
+
+        allclose = duck_array_ops.allclose_or_equiv(
+            actual, desired, rtol=rtol, atol=atol
+        )
+        assert allclose, _format_message(
+            actual, desired, err_msg=err_msg, verbose=verbose
+        )
 
 
 def assert_duckarray_equal(x, y, err_msg="", verbose=True):

xarray/tests/test_testing.py

@@ -1,3 +1,5 @@
+import warnings
+
 import numpy as np
 import pytest
 
@@ -127,3 +129,36 @@ def test_assert_duckarray_equal(duckarray, obj1, obj2):
     b = duckarray(obj2)
 
     xr.testing.assert_duckarray_equal(a, b)
+
+
+@pytest.mark.parametrize(
+    "func",
+    [
+        "assert_equal",
+        "assert_identical",
+        "assert_allclose",
+        "assert_duckarray_allclose",
+    ],
+)
+def test_ensure_warnings_not_elevated(func):
+    # make sure warnings are not elevated to errors in the assertion functions
+    # e.g. by @pytest.mark.filterwarnings("error")
+    # see https://github.com/pydata/xarray/pull/4760#issuecomment-774101639
+
+    # define a custom Variable class that raises a warning in assert_*
+    class WarningVariable(xr.Variable):
+        @property  # type: ignore
+        def dims(self):
+            warnings.warn("warning in test")
+            return super().dims
+
+    a = WarningVariable("x", [1])
+    b = WarningVariable("x", [2])
+
+    with warnings.catch_warnings(record=True) as w:
+        # elevate warnings to errors
+        warnings.filterwarnings("error")
+        with pytest.raises(AssertionError):
+            getattr(xr.testing, func)(a, b)
+
+        assert len(w) > 0