Add fast reductions

docs/src/lib/functions.md

@@ -82,6 +82,7 @@ repeat
 repeat!
 select
 select!
+table_transformation
 transform
 transform!
 vcat

docs/src/lib/internals.md

@@ -16,4 +16,5 @@ getmaxwidths
 ourshow
 ourstrwidth
 @spawn_for_chunks
+default_table_transformation
 ```

docs/src/man/split_apply_combine.md

@@ -79,11 +79,20 @@ as following arguments. The second type of signature is when a `Function` or a `
 is passed as the first argument and a `GroupedDataFrame` as the second argument
 (similar to `map`).
 
-As a special rule, with the `cols => function` and `cols => function =>
-target_cols` syntaxes, if `cols` is wrapped in an `AsTable`
+As a special rule, with the `cols => function` and
+`cols => function => target_cols` syntaxes, if `cols` is wrapped in an `AsTable`
 object then a `NamedTuple` containing columns selected by `cols` is passed to
 `function`.
 
+Note! When `AsTable` is used as source column selector it is possible to
+override the default processing performed by function `function` by adding
+a [`table_transformation`](@ref) method for this function. This is most useful
+for custom reductions over columns of `NamedTuple` created by `AsTable`,
+especially in cases when the user expects that very many (over 1000 as a rule of
+thumb) would be selected by `AsTable` selector in which case avoiding creation
+of `NamedTuple` object significantly reduces compilation time (which is often
+more longer than computation time in such cases).
+
 What is allowed for `function` to return is determined by the `target_cols` value:
 1. If both `cols` and `target_cols` are omitted (so only a `function` is passed),
    then returning a data frame, a matrix, a `NamedTuple`, or a `DataFrameRow` will

src/DataFrames.jl

@@ -129,6 +129,7 @@ include("groupeddataframe/utils.jl")
 include("other/broadcasting.jl")
 
 include("abstractdataframe/selection.jl")
+include("abstractdataframe/selectionfast.jl")
 include("abstractdataframe/subset.jl")
 include("abstractdataframe/iteration.jl")
 include("abstractdataframe/reshape.jl")

src/abstractdataframe/selection.jl

@@ -400,11 +400,11 @@ _transformation_helper(df::AbstractDataFrame, col_idx::Int, (fun,)::Ref{Any}) =
 _empty_astable_helper(fun, len) = [fun(NamedTuple()) for _ in 1:len]
 
 function _transformation_helper(df::AbstractDataFrame, col_idx::AsTable, (fun,)::Ref{Any})
-    tbl = Tables.columntable(select(df, col_idx.cols, copycols=false))
-    if isempty(tbl) && fun isa ByRow
+    df_sel = select(df, col_idx.cols, copycols=false)
+    if ncol(df_sel) == 0 && fun isa ByRow
         return _empty_astable_helper(fun.fun, nrow(df))
     else
-        return fun(tbl)
+        return table_transformation(df_sel, fun)
     end
 end
 
@@ -415,7 +415,17 @@ function _transformation_helper(df::AbstractDataFrame, col_idx::AbstractVector{I
         return _empty_selector_helper(fun.fun, nrow(df))
     else
         cdf = eachcol(df)
-        return fun(map(c -> cdf[c], col_idx)...)
+        cols = map(c -> cdf[c], col_idx)
+        if (fun === +) || fun === ByRow(+)
+            isempty(cols) && return +() # to make sure we produce a consistent error
+            return reduce(+, cols)
+        elseif fun === ByRow(min)
+            return _minmax_row_fast(cols, min)
+        elseif fun === ByRow(max)
+            return _minmax_row_fast(cols, max)
+        else
+            return fun(cols...)
+        end
     end
 end