DEPR: Silent dropping of nuisance columns in agg_list_like

doc/source/user_guide/basics.rst

@@ -1045,6 +1045,9 @@ not noted for a particular column will be ``NaN``:
 Mixed dtypes
 ++++++++++++
 
+.. deprecated:: 1.4.0
+   Attempting to determine which columns cannot be aggregated and silently dropping them from the results is deprecated and will be removed in a future version. If any porition of the columns or operations provided fail, the call to ``.agg`` will raise.
+
 When presented with mixed dtypes that cannot aggregate, ``.agg`` will only take the valid
 aggregations. This is similar to how ``.groupby.agg`` works.
 

doc/source/user_guide/groupby.rst

@@ -578,7 +578,7 @@ column, which produces an aggregated result with a hierarchical index:
 
 .. ipython:: python
 
-   grouped.agg([np.sum, np.mean, np.std])
+   grouped[["C", "D"]].agg([np.sum, np.mean, np.std])
 
 
 The resulting aggregations are named for the functions themselves. If you
@@ -597,7 +597,7 @@ For a grouped ``DataFrame``, you can rename in a similar manner:
 .. ipython:: python
 
    (
-       grouped.agg([np.sum, np.mean, np.std]).rename(
+       grouped[["C", "D"]].agg([np.sum, np.mean, np.std]).rename(
            columns={"sum": "foo", "mean": "bar", "std": "baz"}
        )
    )

doc/source/whatsnew/v0.20.0.rst

@@ -105,6 +105,7 @@ aggregations. This is similar to how groupby ``.agg()`` works. (:issue:`15015`)
    df.dtypes
 
 .. ipython:: python
+   :okwarning:
 
    df.agg(['min', 'sum'])
 

doc/source/whatsnew/v1.4.0.rst

@@ -337,6 +337,7 @@ Other Deprecations
 - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`)
 - Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`)
 - Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`)
+- Deprecated silent dropping of columns that raised a ``TypeError``, ``DataError``, or some cases of ``ValueError`` in :class:`Series.aggregate` and :class:`DataFrame.aggregate` when used with a list (:issue:`43740`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/apply.py

@@ -33,6 +33,7 @@
     FrameOrSeries,
 )
 from pandas.util._decorators import cache_readonly
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.cast import is_nested_object
 from pandas.core.dtypes.common import (
@@ -335,6 +336,7 @@ def agg_list_like(self) -> DataFrame | Series:
 
         results = []
         keys = []
+        failed_names = []
 
         # degenerate case
         if selected_obj.ndim == 1:
@@ -344,7 +346,7 @@ def agg_list_like(self) -> DataFrame | Series:
                     new_res = colg.aggregate(a)
 
                 except TypeError:
-                    pass
+                    failed_names.append(com.get_callable_name(a) or a)
                 else:
                     results.append(new_res)
 
@@ -358,20 +360,23 @@ def agg_list_like(self) -> DataFrame | Series:
             for index, col in enumerate(selected_obj):
                 colg = obj._gotitem(col, ndim=1, subset=selected_obj.iloc[:, index])
                 try:
-                    new_res = colg.aggregate(arg)
+                    with warnings.catch_warnings(record=True) as w:
+                        new_res = colg.aggregate(arg)
+                        if len(w) > 0:
+                            failed_names.append(col)
                 except (TypeError, DataError):
-                    pass
+                    failed_names.append(col)
                 except ValueError as err:
                     # cannot aggregate
                     if "Must produce aggregated value" in str(err):
                         # raised directly in _aggregate_named
-                        pass
+                        failed_names.append(col)
                     elif "no results" in str(err):
                         # reached in test_frame_apply.test_nuiscance_columns
                         #  where the colg.aggregate(arg) ends up going through
                         #  the selected_obj.ndim == 1 branch above with arg == ["sum"]
                         #  on a datetime64[ns] column
-                        pass
+                        failed_names.append(col)
                     else:
                         raise
                 else:
@@ -384,6 +389,15 @@ def agg_list_like(self) -> DataFrame | Series:
         if not len(results):
             raise ValueError("no results")
 
+        if len(failed_names) > 0:
+            warnings.warn(
+                f"{failed_names} did not aggregate successfully. If any error is "
+                f"raised this will raise in a future version of pandas. "
+                f"Drop these columns/ops to avoid this warning.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
+
         try:
             concatenated = concat(results, keys=keys, axis=1, sort=False)
         except TypeError as err:

pandas/tests/apply/test_frame_apply.py

@@ -1087,12 +1087,16 @@ def test_agg_multiple_mixed_no_warning():
         index=["min", "sum"],
     )
     # sorted index
-    with tm.assert_produces_warning(None):
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['D'\] did not aggregate successfully"
+    ):
         result = mdf.agg(["min", "sum"])
 
     tm.assert_frame_equal(result, expected)
 
-    with tm.assert_produces_warning(None):
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['D'\] did not aggregate successfully"
+    ):
         result = mdf[["D", "C", "B", "A"]].agg(["sum", "min"])
 
     # GH40420: the result of .agg should have an index that is sorted
@@ -1201,7 +1205,10 @@ def test_nuiscance_columns():
     expected = Series([6, 6.0, "foobarbaz"], index=["A", "B", "C"])
     tm.assert_series_equal(result, expected)
 
-    result = df.agg(["sum"])
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['D'\] did not aggregate successfully"
+    ):
+        result = df.agg(["sum"])
     expected = DataFrame(
         [[6, 6.0, "foobarbaz"]], index=["sum"], columns=["A", "B", "C"]
     )
@@ -1433,7 +1440,10 @@ def foo(s):
         return s.sum() / 2
 
     aggs = ["sum", foo, "count", "min"]
-    result = df.agg(aggs)
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['item'\] did not aggregate successfully"
+    ):
+        result = df.agg(aggs)
     expected = DataFrame(
         {
             "item": ["123456", np.nan, 6, "1"],

pandas/tests/groupby/aggregate/test_aggregate.py

@@ -339,8 +339,14 @@ def test_multiple_functions_tuples_and_non_tuples(df):
     expected = df.groupby("A")["C"].agg(ex_funcs)
     tm.assert_frame_equal(result, expected)
 
-    result = df.groupby("A").agg(funcs)
-    expected = df.groupby("A").agg(ex_funcs)
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['B'\] did not aggregate successfully"
+    ):
+        result = df.groupby("A").agg(funcs)
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['B'\] did not aggregate successfully"
+    ):
+        expected = df.groupby("A").agg(ex_funcs)
     tm.assert_frame_equal(result, expected)
 
 

pandas/tests/groupby/aggregate/test_other.py

@@ -45,13 +45,15 @@ def peak_to_peak(arr):
         return arr.max() - arr.min()
 
     with tm.assert_produces_warning(
-        FutureWarning, match="Dropping invalid", check_stacklevel=False
+        FutureWarning,
+        match=r"\['key2'\] did not aggregate successfully",
     ):
         expected = grouped.agg([peak_to_peak])
     expected.columns = ["data1", "data2"]
 
     with tm.assert_produces_warning(
-        FutureWarning, match="Dropping invalid", check_stacklevel=False
+        FutureWarning,
+        match=r"\['key2'\] did not aggregate successfully",
     ):
         result = grouped.agg(peak_to_peak)
     tm.assert_frame_equal(result, expected)

pandas/tests/groupby/test_groupby.py

@@ -583,7 +583,10 @@ def test_frame_multi_key_function_list():
 
     grouped = data.groupby(["A", "B"])
     funcs = [np.mean, np.std]
-    agged = grouped.agg(funcs)
+    with tm.assert_produces_warning(
+        FutureWarning, match=r"\['C'\] did not aggregate successfully"
+    ):
+        agged = grouped.agg(funcs)
     expected = pd.concat(
         [grouped["D"].agg(funcs), grouped["E"].agg(funcs), grouped["F"].agg(funcs)],
         keys=["D", "E", "F"],

pandas/tests/resample/test_resample_api.py

@@ -350,10 +350,12 @@ def test_agg():
     expected = pd.concat([a_mean, a_std, b_mean, b_std], axis=1)
     expected.columns = pd.MultiIndex.from_product([["A", "B"], ["mean", "std"]])
     for t in cases:
-        with tm.assert_produces_warning(None):
-            # .var on dt64 column raises and is dropped, but the path in core.apply
-            #  that it goes through will still suppress a TypeError even
-            #  once the deprecations in the groupby code are enforced
+        warn = FutureWarning if t in cases[1:3] else None
+        with tm.assert_produces_warning(
+            warn,
+            match=r"\['date'\] did not aggregate successfully",
+        ):
+            # .var on dt64 column raises and is dropped
             result = t.aggregate([np.mean, np.std])
         tm.assert_frame_equal(result, expected)