Add fast reductions

docs/src/lib/functions.md

@@ -82,6 +82,7 @@ repeat
 repeat!
 select
 select!
+table_transformation
 transform
 transform!
 vcat

docs/src/lib/internals.md

@@ -16,4 +16,15 @@ getmaxwidths
 ourshow
 ourstrwidth
 @spawn_for_chunks
+default_table_transformation
 ```
+
+Note! When `AsTable` is used as source column selector in the
+`source => function => target` mini-language supported by `select` and related
+functions it is possible to override the default processing performed by
+function `function` by adding a [`table_transformation`](@ref) method for this
+function. This is most useful for custom reductions over columns of `NamedTuple`
+created by `AsTable`, especially in cases when the user expects that very many
+(over 1000 as a rule of thumb) would be selected by `AsTable` selector in which
+case avoiding creation of `NamedTuple` object significantly reduces compilation
+time (which is often more longer than computation time in such cases).

docs/src/man/split_apply_combine.md

@@ -79,8 +79,8 @@ as following arguments. The second type of signature is when a `Function` or a `
 is passed as the first argument and a `GroupedDataFrame` as the second argument
 (similar to `map`).
 
-As a special rule, with the `cols => function` and `cols => function =>
-target_cols` syntaxes, if `cols` is wrapped in an `AsTable`
+As a special rule, with the `cols => function` and
+`cols => function => target_cols` syntaxes, if `cols` is wrapped in an `AsTable`
 object then a `NamedTuple` containing columns selected by `cols` is passed to
 `function`.
 

src/DataFrames.jl

@@ -129,6 +129,7 @@ include("groupeddataframe/utils.jl")
 include("other/broadcasting.jl")
 
 include("abstractdataframe/selection.jl")
+include("abstractdataframe/selectionfast.jl")
 include("abstractdataframe/subset.jl")
 include("abstractdataframe/iteration.jl")
 include("abstractdataframe/reshape.jl")

src/abstractdataframe/selection.jl

@@ -48,8 +48,8 @@ const TRANSFORMATION_COMMON_RULES =
        the target column or columns, which must be a single name (as a `Symbol` or a string),
        a vector of names or `AsTable`. Additionally it can be a `Function` which
        takes a string or a vector of strings as an argument containing names of columns
-   selected by `cols`, and returns the target columns names (all accepted types
-   except `AsTable` are allowed).
+       selected by `cols`, and returns the target columns names (all accepted types
+       except `AsTable` are allowed).
     4. a `col => target_cols` pair, which renames the column `col` to `target_cols`, which
        must be single name (as a `Symbol` or a string), a vector of names or `AsTable`.
     5. a `nrow` or `nrow => target_cols` form which efficiently computes the number of rows
@@ -62,6 +62,9 @@ const TRANSFORMATION_COMMON_RULES =
        is small or a very large number of columns are processed
        (in which case `SubDataFrame` avoids excessive compilation)
 
+    In order to improve the performance of the operations some transformations
+    invoke optimized implementation, see [`table_transformation`](@ref) for details.
+
     Note! If the expression of the form `x => y` is passed then except for the special
     convenience form `nrow => target_cols` it is always interpreted as
     `cols => function`. In particular the following expression `function => target_cols`
@@ -400,11 +403,11 @@ _transformation_helper(df::AbstractDataFrame, col_idx::Int, (fun,)::Ref{Any}) =
 _empty_astable_helper(fun, len) = [fun(NamedTuple()) for _ in 1:len]
 
 function _transformation_helper(df::AbstractDataFrame, col_idx::AsTable, (fun,)::Ref{Any})
-    tbl = Tables.columntable(select(df, col_idx.cols, copycols=false))
-    if isempty(tbl) && fun isa ByRow
+    df_sel = select(df, col_idx.cols, copycols=false)
+    if ncol(df_sel) == 0 && fun isa ByRow
         return _empty_astable_helper(fun.fun, nrow(df))
     else
-        return fun(tbl)
+        return table_transformation(df_sel, fun)
     end
 end
 
@@ -415,7 +418,17 @@ function _transformation_helper(df::AbstractDataFrame, col_idx::AbstractVector{I
         return _empty_selector_helper(fun.fun, nrow(df))
     else
         cdf = eachcol(df)
-        return fun(map(c -> cdf[c], col_idx)...)
+        cols = map(c -> cdf[c], col_idx)
+        if (fun === +) || fun === ByRow(+) # removing parentheses leads to a parsing error
+            isempty(cols) && return +() # to make sure we produce a consistent error
+            return reduce(+, cols)
+        elseif fun === ByRow(min)
+            return _minmax_row_fast(cols, min)
+        elseif fun === ByRow(max)
+            return _minmax_row_fast(cols, max)
+        else
+            return fun(cols...)
+        end
     end
 end