From c3ac291d7d8c23d32c4d281d9595b1d59ceae594 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 17 Apr 2018 16:48:58 +0200 Subject: [PATCH 1/9] DEPR: Series ndarray properties (strides, data, base, itemsize, flags) --- pandas/core/base.py | 15 +++++++++++++++ pandas/core/indexes/datetimelike.py | 10 ++++++++++ pandas/tests/test_base.py | 18 ++++++++++++++---- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pandas/core/base.py b/pandas/core/base.py index 0d55fa8b97aae..5a9711391c7c8 100644 --- a/pandas/core/base.py +++ b/pandas/core/base.py @@ -737,11 +737,17 @@ def item(self): @property def data(self): """ return the data pointer of the underlying data """ + warnings.warn("Series/Index.data is deprecated and will be " + "removed in a future version", + FutureWarning, stacklevel=2) return self.values.data @property def itemsize(self): """ return the size of the dtype of the item of the underlying data """ + warnings.warn("Series/Index.itemsize is deprecated and will be " + "removed in a future version", + FutureWarning, stacklevel=2) return self._ndarray_values.itemsize @property @@ -752,6 +758,9 @@ def nbytes(self): @property def strides(self): """ return the strides of the underlying data """ + warnings.warn("Series/Index.strides is deprecated and will be " + "removed in a future version", + FutureWarning, stacklevel=2) return self._ndarray_values.strides @property @@ -762,6 +771,9 @@ def size(self): @property def flags(self): """ return the ndarray.flags for the underlying data """ + warnings.warn("Series/Index.flags is deprecated and will be " + "removed in a future version", + FutureWarning, stacklevel=2) return self.values.flags @property @@ -769,6 +781,9 @@ def base(self): """ return the base object if the memory of the underlying data is shared """ + warnings.warn("Series/Index.base is deprecated and will be " + "removed in a future version", + FutureWarning, stacklevel=2) return self.values.base @property diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 95e1f8438c704..432c1c67cb86c 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -209,6 +209,16 @@ def ceil(self, freq): class DatetimeIndexOpsMixin(object): """ common ops mixin to support a unified interface datetimelike Index """ + @property + def base(self): + """ return the base object if the memory of the underlying data is + shared + """ + # override deprecated property in IndexOpsMixin, as we still need + # this for internals (DatetimeIndex/TimedeltaIndex is stored as + # values in Blocks) + return self.values.base + def equals(self, other): """ Determines if two Index objects contain the same elements. diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index c4c02c0bf6f17..9a7157dd06b25 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -22,6 +22,7 @@ from pandas.core.base import PandasObject, NoNewAttributesMixin from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin from pandas._libs.tslib import iNaT +import pandas.util.testing as tm class CheckStringMixin(object): @@ -316,16 +317,25 @@ def test_ndarray_compat_properties(self): for o in self.objs: # Check that we work. - for p in ['shape', 'dtype', 'flags', 'T', - 'strides', 'itemsize', 'nbytes']: + for p in ['shape', 'dtype', 'T', 'nbytes']: assert getattr(o, p, None) is not None - assert hasattr(o, 'base') + # deprecated properties + for p in ['flags', 'strides', 'itemsize']: + with tm.assert_produces_warning(FutureWarning): + assert getattr(o, p, None) is not None + + # not deprecated for datetime-like indices because they are used + # inside blocks + if not isinstance(o, (DatetimeIndex, TimedeltaIndex, PeriodIndex)): + with tm.assert_produces_warning(FutureWarning): + assert hasattr(o, 'base') # If we have a datetime-like dtype then needs a view to work # but the user is responsible for that try: - assert o.data is not None + with tm.assert_produces_warning(FutureWarning): + assert o.data is not None except ValueError: pass From 11182b196268e750718f4582604f74164af234d4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 18 Apr 2018 10:43:14 +0200 Subject: [PATCH 2/9] better warning message --- pandas/core/base.py | 20 ++++++++++---------- pandas/tests/test_base.py | 1 - 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pandas/core/base.py b/pandas/core/base.py index 5a9711391c7c8..9ca1c8bea4db7 100644 --- a/pandas/core/base.py +++ b/pandas/core/base.py @@ -737,16 +737,16 @@ def item(self): @property def data(self): """ return the data pointer of the underlying data """ - warnings.warn("Series/Index.data is deprecated and will be " - "removed in a future version", + warnings.warn("{obj}.data is deprecated and will be removed " + "in a future version".format(obj=type(self).__name__), FutureWarning, stacklevel=2) return self.values.data @property def itemsize(self): """ return the size of the dtype of the item of the underlying data """ - warnings.warn("Series/Index.itemsize is deprecated and will be " - "removed in a future version", + warnings.warn("{obj}.itemsize is deprecated and will be removed " + "in a future version".format(obj=type(self).__name__), FutureWarning, stacklevel=2) return self._ndarray_values.itemsize @@ -758,8 +758,8 @@ def nbytes(self): @property def strides(self): """ return the strides of the underlying data """ - warnings.warn("Series/Index.strides is deprecated and will be " - "removed in a future version", + warnings.warn("{obj}.strudes is deprecated and will be removed " + "in a future version".format(obj=type(self).__name__), FutureWarning, stacklevel=2) return self._ndarray_values.strides @@ -771,8 +771,8 @@ def size(self): @property def flags(self): """ return the ndarray.flags for the underlying data """ - warnings.warn("Series/Index.flags is deprecated and will be " - "removed in a future version", + warnings.warn("{obj}.flags is deprecated and will be removed " + "in a future version".format(obj=type(self).__name__), FutureWarning, stacklevel=2) return self.values.flags @@ -781,8 +781,8 @@ def base(self): """ return the base object if the memory of the underlying data is shared """ - warnings.warn("Series/Index.base is deprecated and will be " - "removed in a future version", + warnings.warn("{obj}.base is deprecated and will be removed " + "in a future version".format(obj=type(self).__name__), FutureWarning, stacklevel=2) return self.values.base diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 9a7157dd06b25..c7583470e6e46 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -22,7 +22,6 @@ from pandas.core.base import PandasObject, NoNewAttributesMixin from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin from pandas._libs.tslib import iNaT -import pandas.util.testing as tm class CheckStringMixin(object): From 1aeaf2eca20f44af0f8ec8d04a0d2c35bd11cf56 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 18 Apr 2018 10:47:36 +0200 Subject: [PATCH 3/9] add whatsnew --- doc/source/whatsnew/v0.23.0.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 641214550a3b7..1a1234e73062a 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -877,6 +877,9 @@ Deprecations - The ``convert_datetime64`` parameter in :func:`DataFrame.to_records` has been deprecated and will be removed in a future version. The NumPy bug motivating this parameter has been resolved. The default value for this parameter has also changed from ``True`` to ``None`` (:issue:`18160`). - :func:`Series.rolling().apply() `, :func:`DataFrame.rolling().apply() `, :func:`Series.expanding().apply() `, and :func:`DataFrame.expanding().apply() ` have deprecated passing an ``np.array`` by default. One will need to pass the new ``raw`` parameter to be explicit about what is passed (:issue:`20584`) +- The ``data``, ``base``, ``strides``, ``flags`` and ``itemsize`` properties + of the ``Series`` and ``Index`` classes have been deprecated and will be + removed in a future version (:issue:`20419`). .. _whatsnew_0230.prior_deprecations: From 5339975357731add803c046efcc5934542b04bc4 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 18 Apr 2018 13:24:07 +0200 Subject: [PATCH 4/9] fix usage in tests --- pandas/tests/groupby/aggregate/test_other.py | 2 +- pandas/tests/groupby/test_groupby.py | 2 +- pandas/tests/indexes/common.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/groupby/aggregate/test_other.py b/pandas/tests/groupby/aggregate/test_other.py index 7c6cb5b9615cb..a10f7f6e46210 100644 --- a/pandas/tests/groupby/aggregate/test_other.py +++ b/pandas/tests/groupby/aggregate/test_other.py @@ -328,7 +328,7 @@ def test_series_agg_multi_pure_python(): 'F': np.random.randn(11)}) def bad(x): - assert (len(x.base) > 0) + assert (len(x.values.base) > 0) return 'foo' result = data.groupby(['A', 'B']).agg(bad) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index cdb4e3072c65d..3c4a89c0145c5 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1618,7 +1618,7 @@ def convert_fast(x): def convert_force_pure(x): # base will be length 0 - assert (len(x.base) > 0) + assert (len(x.values.base) > 0) return Decimal(str(x.mean())) grouped = s.groupby(labels) diff --git a/pandas/tests/indexes/common.py b/pandas/tests/indexes/common.py index 758f3f0ef9ebc..f78bd583288a4 100644 --- a/pandas/tests/indexes/common.py +++ b/pandas/tests/indexes/common.py @@ -24,7 +24,7 @@ class Base(object): """ base class for index sub-class tests """ _holder = None - _compat_props = ['shape', 'ndim', 'size', 'itemsize', 'nbytes'] + _compat_props = ['shape', 'ndim', 'size', 'nbytes'] def setup_indices(self): for name, idx in self.indices.items(): From c78d5d698623fbd758767a2a2dc18ac8f96e2336 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 19 Apr 2018 14:31:58 +0200 Subject: [PATCH 5/9] fix remaining tests --- pandas/core/algorithms.py | 2 ++ pandas/tests/indexes/test_multi.py | 2 +- pandas/tests/indexes/test_range.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 065a5782aced1..f2b0cdc0705cd 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -1491,6 +1491,8 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None, if is_sparse(arr): arr = arr.get_values() + arr = np.array(arr, copy=False) + if indexer is None: indexer = np.arange(arr.shape[axis], dtype=np.int64) dtype, fill_value = arr.dtype, arr.dtype.type() diff --git a/pandas/tests/indexes/test_multi.py b/pandas/tests/indexes/test_multi.py index 984f37042d600..e189b6e856cb7 100644 --- a/pandas/tests/indexes/test_multi.py +++ b/pandas/tests/indexes/test_multi.py @@ -30,7 +30,7 @@ class TestMultiIndex(Base): _holder = MultiIndex - _compat_props = ['shape', 'ndim', 'size', 'itemsize'] + _compat_props = ['shape', 'ndim', 'size'] def setup_method(self, method): major_axis = Index(['foo', 'bar', 'baz', 'qux']) diff --git a/pandas/tests/indexes/test_range.py b/pandas/tests/indexes/test_range.py index 1ebeef072fdc5..8990834ebe91a 100644 --- a/pandas/tests/indexes/test_range.py +++ b/pandas/tests/indexes/test_range.py @@ -22,7 +22,7 @@ class TestRangeIndex(Numeric): _holder = RangeIndex - _compat_props = ['shape', 'ndim', 'size', 'itemsize'] + _compat_props = ['shape', 'ndim', 'size'] def setup_method(self, method): self.indices = dict(index=RangeIndex(0, 20, 2, name='foo'), From 160ace0281c0d697313dd16f0db0ce6794f69814 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 23 Apr 2018 09:44:09 +0200 Subject: [PATCH 6/9] also deprecate DatetimeIndex.base --- pandas/core/indexes/datetimelike.py | 10 ---------- pandas/core/internals.py | 6 ++++++ pandas/tests/test_base.py | 7 ++----- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index b36720a2c1afb..95186b2e79a16 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -209,16 +209,6 @@ def ceil(self, freq): class DatetimeIndexOpsMixin(object): """ common ops mixin to support a unified interface datetimelike Index """ - @property - def base(self): - """ return the base object if the memory of the underlying data is - shared - """ - # override deprecated property in IndexOpsMixin, as we still need - # this for internals (DatetimeIndex/TimedeltaIndex is stored as - # values in Blocks) - return self.values.base - def equals(self, other): """ Determines if two Index objects contain the same elements. diff --git a/pandas/core/internals.py b/pandas/core/internals.py index e98899b2f5c1a..a266ea620bd9f 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -2816,6 +2816,12 @@ def _maybe_coerce_values(self, values, dtype=None): return values + @property + def is_view(self): + """ return a boolean if I am possibly a view """ + # check the ndarray values of the DatetimeIndex values + return self.values.values.base is not None + def copy(self, deep=True, mgr=None): """ copy constructor """ values = self.values diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index c7583470e6e46..a5d83c1c26948 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -324,11 +324,8 @@ def test_ndarray_compat_properties(self): with tm.assert_produces_warning(FutureWarning): assert getattr(o, p, None) is not None - # not deprecated for datetime-like indices because they are used - # inside blocks - if not isinstance(o, (DatetimeIndex, TimedeltaIndex, PeriodIndex)): - with tm.assert_produces_warning(FutureWarning): - assert hasattr(o, 'base') + with tm.assert_produces_warning(FutureWarning): + assert hasattr(o, 'base') # If we have a datetime-like dtype then needs a view to work # but the user is responsible for that From d2088d275d46925db8ad884d1938a97e6f3c2012 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 23 Apr 2018 09:44:36 +0200 Subject: [PATCH 7/9] array -> asarray --- pandas/core/algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index f2b0cdc0705cd..491d8a628f1d6 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -1491,7 +1491,7 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None, if is_sparse(arr): arr = arr.get_values() - arr = np.array(arr, copy=False) + arr = np.asarray(arr) if indexer is None: indexer = np.arange(arr.shape[axis], dtype=np.int64) From 7687330942d4ed0c3f7c0d2a4ef311e02ca7924e Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 24 Apr 2018 14:33:24 +0200 Subject: [PATCH 8/9] fix usage of base in DatetimeIndexConstructor (_deepcopy_if_needed) --- pandas/core/indexes/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 12bb09e8f8a8a..734b002691d1c 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -548,6 +548,9 @@ def _deepcopy_if_needed(self, orig, copy=False): """ if copy: # Retrieve the "base objects", i.e. the original memory allocations + if not isinstance(orig, np.ndarray): + # orig is a DatetimeIndex + orig = orig.values orig = orig if orig.base is None else orig.base new = self._data if self._data.base is None else self._data.base if orig is new: From feb7caf297a7867d344576eee352b34f34f429a9 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 24 Apr 2018 14:42:09 +0200 Subject: [PATCH 9/9] fix usage of .base in BlockManager test_copy --- pandas/tests/internals/test_internals.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index 9338aba90d7cb..7fbf7ec05e91e 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -466,7 +466,11 @@ def test_copy(self, mgr): # view assertion assert cp_blk.equals(blk) - assert cp_blk.values.base is blk.values.base + if isinstance(blk.values, np.ndarray): + assert cp_blk.values.base is blk.values.base + else: + # DatetimeTZBlock has DatetimeIndex values + assert cp_blk.values.values.base is blk.values.values.base cp = mgr.copy(deep=True) for blk, cp_blk in zip(mgr.blocks, cp.blocks): @@ -474,8 +478,8 @@ def test_copy(self, mgr): # copy assertion we either have a None for a base or in case of # some blocks it is an array (e.g. datetimetz), but was copied assert cp_blk.equals(blk) - if cp_blk.values.base is not None and blk.values.base is not None: - assert cp_blk.values.base is not blk.values.base + if not isinstance(cp_blk.values, np.ndarray): + assert cp_blk.values.values.base is not blk.values.values.base else: assert cp_blk.values.base is None and blk.values.base is None