diff --git a/collapse.pdf b/collapse.pdf index 94cd1225..e83f05ae 100644 Binary files a/collapse.pdf and b/collapse.pdf differ diff --git a/latex/collapse/background_cropped.png b/latex/collapse/background_cropped.png new file mode 100644 index 00000000..961b5a62 Binary files /dev/null and b/latex/collapse/background_cropped.png differ diff --git a/latex/collapse/collapse_cheat_sheet.Rnw b/latex/collapse/collapse_cheat_sheet.Rnw new file mode 100644 index 00000000..afa8b7c8 --- /dev/null +++ b/latex/collapse/collapse_cheat_sheet.Rnw @@ -0,0 +1,697 @@ +\documentclass[8pt,a4paper,landscape]{extarticle} + +%% Language and font encodings +\usepackage[english]{babel} +\usepackage[utf8x]{inputenc} +% \usepackage[T1]{fontenc} + +%% Sets page size and margins +\usepackage[landscape,top=0.5cm,bottom=-1cm,left=0.5cm,right=0.5cm,marginparwidth=1.75cm]{geometry} + +%% Useful packages +\usepackage{amsmath} +\usepackage{multicol} % # ,multirow +\setlength{\columnsep}{0.5cm} +\usepackage{graphicx} +\usepackage[dvipsnames]{xcolor} +% \usepackage[colorinlistoftodos]{todonotes} +\usepackage[colorlinks=true, allcolors=NavyBlue]{hyperref} +%\usepackage[section]{placeins} +\usepackage{booktabs} +\usepackage[scaled=1]{helvet} +\renewcommand\familydefault{\sfdefault} +\renewcommand\bfdefault{sb} +% \renewcommand\bfdefault{md} +%\usepackage{fontspec} +% \usepackage{enumitem} +\usepackage{scrextend} % https://tex.stackexchange.com/questions/37080/how-can-i-indent-a-block-of-text-for-a-specified-amount +\usepackage{anyfontsize} % \usepackage{moresize} % https://texblog.org/2012/08/29/changing-the-font-size-in-latex/ + +\usepackage{setspace} % \linespread{1.3} +\setcounter{secnumdepth}{0} +\setlength{\parindent}{0pt} +\setlength{\parskip}{0.02em}%Length of space between paragraphs +% \setlength{\columnsep}{6mm} +\setlength{\itemsep}{0.2em} +\pagestyle{empty} +% \usepackage{fancyhdr} +% \rfoot{Page \thepage} + +\makeatletter +\renewcommand{\section}{\@startsection{section}{1}{0mm}% + {-2ex plus -.5ex minus -.2ex}%before -1 + {0.5ex plus .2ex}%x + {\normalfont\large\bfseries\textcolor{WildStrawberry}}} +\renewcommand{\subsection}{\@startsection{subsection}{2}{0mm}% + {-2explus -.5ex minus -.2ex}%before -1 + {0.5ex plus .2ex}% + {\normalfont\normalsize\bfseries\textcolor{WildStrawberry}}} +\renewcommand{\subsubsection}{\@startsection{subsubsection}{3}{0mm}% + {-2ex plus -.5ex minus -.2ex}%set spaces around sections + {0.3ex plus .2ex}%normally 1 + {\normalfont\small\bfseries\textcolor{WildStrawberry}}} +\makeatother + +% https://tex.stackexchange.com/questions/36030/how-to-make-a-single-word-look-as-some-code +%\definecolor{light-gray}{gray}{0.95} +%\newcommand{\code}[1]{\colorbox{light-gray}{\texttt{#1}}} +\newcommand{\code}[1]{\texttt{#1}} +\newcommand{\hrrule}{\vspace{1mm}\textcolor{gray}{\hrulefill}\vspace{-2.5mm}} % darkgray +\newcommand{\itxt}[1]{\textcolor{gray}{\textbf{#1}}} + + +% https://groups.google.com/g/knitr/c/mgtuTyBzkaA +\renewenvironment{knitrout}{\setlength{\topsep}{0mm}}{} + +% ----------------------------------------------------------------------- + +% https://tex.stackexchange.com/questions/46280/how-to-create-a-background-image-on-title-page-with-latex +\usepackage{eso-pic} +\newcommand\BackgroundPic{% +\put(0,0){% +\parbox[b][\paperheight]{\paperwidth}{% +\vspace{-3mm} % \vfill +\flushright +\includegraphics[width=0.3\paperwidth,% +keepaspectratio]{"background_cropped.png"}% +\vfill +}}} + +\newcommand\Logo{% +\put(0,0){% +\parbox[b][\paperheight]{\paperwidth}{% +\vspace{-1mm} % \vfill +\flushright +\includegraphics[width=0.08\paperwidth,% +keepaspectratio]{"collapse_logo_vsmall.png"} \hspace{3mm} % +\vfill +}}} + + +\begin{document} + +% \makebox[0pt][r]{% +% \raisebox{-\totalheight}[0pt][0pt]{% +% \includegraphics[width=4in]{background}}}% +\AddToShipoutPicture*{\BackgroundPic} +\AddToShipoutPicture*{\Logo} + +<>= +knitr::opts_chunk$set(warning = FALSE, message = FALSE, cache = FALSE, size = "scriptsize") +library(collapse) +library(magrittr) +library(tibble) +library(microbenchmark) +options(width=80) + +iris2 <- copyv(iris, NA, NA) +@ + +\raggedright +% \footnotesize +\small + +{ + {\fontsize{22}{30}\selectfont \textcolor{Gray}{Advanced and Fast Data Transformation with \emph{collapse}}}{\Huge\ \textcolor{darkgray}{: : CHEAT SHEET}} %\\%\small{by Sebastian Krantz} % + % \vspace{2mm} +} + +%\begin{adjustbox}{totalheight=0.5\textheight} % -2\baselineskip +% \resizebox*{!}{\textheight}{% +\begin{multicols}{4} +%\setlength{\premulticols}{1pt} +%\setlength{\postmulticols}{1pt} +%\setlength{\multicolsep}{1pt} +%\setlength{\columnsep}{2pt} + +\section{\textcolor{WildStrawberry}{Introduction}} % Basics / +%\colorbox{gray}{ +\textbf{\emph{collapse}} is a C/C++ based package supporting advanced (grouped, weighted, time series, panel data and recursive) statistical operations in R, with very efficient low-level vectorizations across both groups and columns. \\ [0.8em] + +It also offers a flexible, class-agnostic, approach to data transformation in R: handling matrix and data frame based objects in a uniform, attribute preserving, way, and ensuring seamless compatibility with base R, \emph{dplyr} / (grouped) \emph{tibble}, \emph{data.table}, \emph{xts/zoo}, \emph{sf}, and \emph{plm} classes for panel data. \\ [0.8em] + +\emph{collapse} provides full control to the user for statistical programming - with several ways to reach the same outcome and rich optimization possibilities. It is globally configurable using \code{set\_collapse()} which includes algorithm defaults, multithreading, and the exported namespace (see below). \\ [0.8em] + +Calling \code{help("collapse-documentation")} brings up a detailed documentation, which is also available \href{https://sebkrantz.github.io/collapse/reference/index.html}{online}. See also the \href{https://fastverse.github.io/fastverse/}{\emph{fastverse}} package/project for a recommended set of complimentary packages and easy package management. +%} + +\hrrule +\section{Row/Column Arithmetic (by Reference)} +\setstretch{1.5} +Column-wise sweeping out of vectors/matrices/DFs/lists\\ +\quad \code{\%cr\%}, \code{\%c+\%}, \code{\%c-\%}, \code{\%c*\%}, \code{\%c/\%} e.g. \code{Z = X \%c/\% rowSums(X)}\\ +Row-wise sweeping vectors from vectors/matrices/DFs/lists\\ +\quad \code{\%rr\%}, \code{\%r+\%}, \code{\%r-\%}, \code{\%r*\%}, \code{\%r/\%} e.g. \code{Z = X \%r/\% colSums(X)}\\ +Standard (column-wise) math by reference (returns invisibly)\\ +\quad \code{\%+=\%}, \code{\%-=\%}, \code{\%*=\%}, \code{\%/=\%} \quad e.g. \quad \code{X \%-=\% rowSums(X)}\\ +Same thing, also supports row-wise operations by reference\\ +\quad \code{setop(X, "/", rowSums(X))}\\ +\setstretch{1} +\quad \code{setop(X, "/", colSums(X), rowwise = TRUE)}\\ + + +\hrrule +\section{\fontsize{9}{11}\selectfont Transform Data by (Grouped) Replacing or +Sweeping out Statistics (by Reference)} +%\vspace{1mm} +\itxt{A generalisation of rowwise operations, that also supports sweeping by groups e.g. aggregate statistics}\newline + +\qquad \code{TRA(x, STATS, FUN = "-", g = NULL, set = FALSE)}\\ +\ \code{setTRA(x, STATS, FUN = "-", g = NULL)}\newline + +\begin{addmargin}[1em]{0em}% 2em left, 2em right +\begin{itemize} +\item[\code{x}] vector, matrix, or (grouped) data frame / list +\item[\code{STATS}] statistics matching (columns of) \code{x} (i.e. aggregated vector, matrix or data frame / list) +\item[\code{FUN}] integer/string indicating transformation to perform: \\\vspace{1mm} +\resizebox{0.2\textwidth}{!}{ +\hspace{-13mm} +\begin{tabular}{lll} + \emph{Int.} & \emph{String} & \emph{Description} \\ + 0 & \code{"replace\_NA"} & replace missing values in \code{x} \\ + 1 & \code{"replace\_fill"} & replace data and missing values in \code{x} \\ + 2 & \code{"replace"} & replace data but preserve missing values in \code{x} \\ + 3 & \code{"-"} & subtract: \code{x - STATS(g)} \\ + 4 & \code{"-+"} & \code{x - STATS(g) + fmean(STATS, w = GRPN)} \\ + 5 & \code{"/"} & divide: \code{x / STATS(g)} \\ + 6 & \code{"\%"} & compute percentages: \code{x * 100/STATS(g)} \\ + 7 & \code{"+"} & add: \code{x + STATS(g)} \\ + 8 & \code{"*"} & multiply: \code{x * STATS(g)} \\ + 9 & \code{"\%\%"} & modulus: \code{x \%\% STATS(g)} \\ + 10 & \code{"-\%\%"} & subtract modulus: \code{x - x \%\% STATS(g)} +\end{tabular} +} +\item[\code{g}] [optional] (list of) vectors / factors or \code{GRP()} object +\item[\code{set}] TRUE transforms \code{x} by reference. \code{setTRA} is equivalent to \code{invisible(TRA(..., set = TRUE))} +\end{itemize} +\end{addmargin} + +% \vfill\null +% \columnbreak +% \hrrule + +\section{Fast Statistical Functions} +%\vspace{1mm} +\itxt{Fast functions to perform column–wise grouped and weighted computations on matrix-like objects} +\newline + +\quad \code{fmean, fmedian, fmode, fsum, fprod, fsd, fvar,} \\ +\quad \code{fmin, fmax, fnth, ffirst, flast, fnobs, fndistinct} \newline + +\textbf{Syntax} \newline + +\quad \code{FUN(x, g = NULL, [w = NULL], TRA = NULL,} \\ +\quad\qquad \code{[na.rm = TRUE], use.g.names = TRUE, }\\ +\quad\qquad \code{[drop = TRUE], [nthreads = 1L])} \newline + +\begin{addmargin}[1em]{0em}% 2em left, 0em right +\begin{itemize} +\item[\code{x}] vector, matrix, or (grouped) data frame / list +\item[\code{g}] [optional] (list of) vectors / factors or GRP() object +\item[\code{w}] [optional] vector of (frequency) weights +\item[\code{TRA}] [optional] operation to transform data with computed statistics (see \code{FUN} argument to \code{TRA()} and Examples) +\item[\code{drop}] drop matrix / data frame dimensions. default \code{TRUE} +% \item[\code{na.rm}] default \code{TRUE}: algorithms have branches to skip \code{NA's} +\end{itemize} +\end{addmargin} + +\vspace{2mm} +\textbf{Examples} +<<>>= +fmean(AirPassengers) # Vector +fmean(AirPassengers, w = cycle(AirPassengers)) # Weighted mean +fmean(EuStockMarkets) # Matrix +fmean(airquality) # Data Frame (use drop = FALSE to keep frame) +fmean(iris[1:4], g = iris$Species) # Grouped +X = iris[1:4]; g = iris$Species; w <- abs(rnorm(nrow(X))) +fmean(X, g, w) # Grouped and weighted (random weights) +## Transfomrations: here centering data on the weighted group median +TRA(X, fmedian(X, g, w), "-", g) |> head(2) +fmedian(X, g, w, TRA = "-") |> head(2) # Same thing: more compact +fmedian(X, g, w, "-", set = TRUE) # Modify in-place (same as setTRA()) +@ +% \begin{addmargin}[2em]{0em} +% \code{fmean(data[3:5], data\$grp1, data\$weights)\\ +% data \%>\% fgroup\_by(grp1) \%>\% fmean(weights)\\ +% TRA(mat, fmedian(mat, g), "-", g)\\ +% fmedian(mat, g, TRA = "-") \# same thing +% } +% \end{addmargin} +\vspace{-1mm} +\hrrule +\section{Other Statistical Functions} +% \vspace{1mm} +\itxt{Fast (weighted) sample quantiles, range, and distances}\\ +\setstretch{1.5} +\code{fquantile(x, probs, w, o, na.rm = TRUE, type = 7)}\\ +\code{frange(x, na.rm = TRUE)} \\ +\code{fdist(x, v, method = "euclidean", nthreads = 1)}\\ +\setstretch{1} + +<>= +iris <- iris2 +@ +% \vspace{-1mm} + +\hrrule +\section{Basic Computing with R Functions} +% \vspace{1mm} +\itxt{Apply R functions to rows or columns (by groups)}\\ +\setstretch{1.5} +\code{dapply(x, FUN, ..., MARGIN = 2)} - column/row apply\\ +\code{BY(x, g, FUN, ...)} - split-apply-combine computing\\ +\setstretch{1} + +%\vfill\null +% \columnbreak + +\section{Grouping and Ordering} +% \vspace{1mm} +\itxt{Optimized functions for grouping, ordering, unique values, matching, splitting, and dealing with factors} +\newline + +\code{GRP()} - create a grouping object (class 'GRP'): pass to \code{g} arg. %\newline +<<>>= +g <- GRP(iris, ~ Species) # or GRP(iris$Species) or GRP(iris["Species"]) +fndistinct(iris[1:4], g) # Computation without grouping overhead +@ + +\code{fgroup\_by()} - attach 'GRP' object to data: a class-agnostic\\ \hphantom{\code{fgroup\_by()} -} grouped frame supporting fast computations + +<<>>= +mtcars |> fgroup_by(cyl, vs, am) |> ss(1:2) +# Group Stats: [N. groups | mean (sd) min-max of group sizes] +# Fast Functions also have a grouped_df method: here wt-weighted medians +mtcars |> fgroup_by(cyl, vs, am) |> fmedian(wt) |> head(2) +@ +%\qquad {\scriptsize \textcolor{darkgray}{\emph{Group Stats:} N. groups $|$ Mean (Std. Dev.) Min-Max of group sizes}} \newline + +\code{GRPN(), fcount[v](), fgroup\_vars(), fungroup()} - get group count, grouping columns, and ungroup data\\ [0.5em] +\code{qF(), qG()} - quick \code{as.factor}, and vector grouping object\\ \qquad of class 'qG': a factor-light without levels attribute\\ +\setstretch{1.5} +\code{group()} - (multivariate) group id ('qG') in appearance order\\ +\code{groupid()} - run-length-type group id ('qG')\\ +\code{seqid()} - group-id from integer-sequences ('qG')\\ +% \code{timeid()} - time variable to integer by GCD (class 'qG')\\ +\code{radixorder[v]()} - (multivariate) radix-based ordering\\ +\code{finteraction()} - fast factor interactions (or return 'qG')\\ +\code{fdroplevels()} - fast removal of unused factor levels\\ +\code{f[n]unique(), fduplicated()} - fast unique values / rows\\ +\code{fmatch(), \%[!][i]in\%} - fast matching of values / rows\\ +\code{gsplit()} - fast splitting vector based on 'GRP' objects\\ +\code{greorder()} - efficiently reorder \code{y = unlist(gsplit(x, g))}\\ \qquad such that \code{identical(greorder(y, g), x)} +\setstretch{1} +\newline + +\emph{collapse} optimizes grouping using both factors / 'qG' objects and 'GRP' objects. 'GRP' objects contain most information and are thus most efficient for complex computations. + +<>= +X <- iris[1:4]; v <- as.character(iris$Species) +f <- qF(v, na.exclude = FALSE) # Adds 'na.included' class: no NA checks +gv <- group(v) # 'qG' object: first appearance order, with 'na.included' +microbenchmark(fmode(X, v), fmode(X, f), fmode(X, gv), fmode(X, g)) +@ +\vspace{-1mm} + +\hrrule +\vspace{-2mm} +\section{Quick Conversions} +\itxt{Fast and exact conversion of common data objects} \\ [0.5em] +\code{qM(), qDF(), qDT(), qTBL()} - convert vectors, arrays, data.frames or lists to matrix, data.frame, data.table or tibble\\ [0.5em] +\code{m[r|c]tl()} - matrix rows/cols to list, data.frame or data.table\\ [0.5em] +\code{qF(), as\_numeric\_factor(), as\_character\_factor()} - convert to/from factors or all factors in a list / data.frame + +% \vfill\null +% \columnbreak +% \newpage + +\section{Fast Data Manipulation} +\vspace{-1.4mm} +\itxt{Minimal overhead implementations} \newline +\setstretch{1.5} +\code{fselect[<-]()} - select/replace columns\\ +\code{fsubset()} - subset data (rows and columns)\\ +\code{ss()} - fast alternative to \code{[}, particularly for data frames\\ +\code{[row|col]order[v]()} - reorder (sort) rows and columns\\ +\code{fmutate(), fsummarise()} - \emph{dplyr}-like, incl. \code{across()} feature\\ +\code{[f|set]transform[v][<-]()} - transform cols (by reference)\\ +\code{fcompute[v]()} - compute new cols dropping existing ones\\ +\code{[f|set]rename()} - rename (any object with 'names' attribute)\\ +\code{[set]relabel()} - assign/change variable labels ('label' attr.)\\ +\code{get\_vars[<-]()} - select/replace columns (standard eval.)\\ [0.5em] +\setstretch{1} +\code{[num|cat|char|fact|logi|date]\_vars[<-]()} - select/\\ \qquad replace columns by data type or retrieve names/indices\\ [0.5em] +\code{add\_vars[<-]()} - add or column-bind columns\\ +\code{rowbind()} - row-bind lists / data frame-like objects\\ [0.5em] +\code{join(), pivot()} - join and reshape data frame-like objects \newline + +\textbf{Examples} +<<>>= +mtcars |> fsubset(mpg > fnth(mpg, 0.95), disp:wt, cylinders = cyl) +mtcars |> colorder(cyl, vs, am, pos = 'after') |> head(2) +i <- base::invisible # These are equivalent, the second option is faster: +mtcars |> fgroup_by(cyl, vs, am) |> fmutate(sum_mpg = fsum(mpg)) |> i() +mtcars |> fmutate(sum_mpg = fsum(mpg, list(cyl, vs, am), TRA = 1)) |> i() +# These are also equivalent (weighted means), again the second is faster +mtcars |> fgroup_by(cyl) |> fmutate(across(disp:drat, fmean, wt)) |> i() +mtcars |> ftransformv(disp:drat, fmean, cyl, wt, 1, apply = FALSE) |> i() +# ftransform()/fcompute() support list input and ignore attached groupings +mtcars %>% fgroup_by(cyl) %>% ftransform(fselect(., hp:qsec) %>% + fmedian(TRA = 1) %>% fungroup() %>% fsum(TRA = "/")) |> i() +# Again a faster equivalent: note the use of 'set' to avoid a deep copy +mtcars %>% ftransform(fselect(., hp:qsec) %>% fmedian(cyl, TRA = 1) %>% + fsum(TRA = "/", set = TRUE)) %>% i() +# Aggregation: weighted standard deviations +mtcars |> fgroup_by(vs) |> fsummarise(across(disp:drat, fsd, w = wt)) +# Grouped linear models (one way of doing it) +qTBL(mtcars) |> fgroup_by(vs) |> fsummarise(reg = list(lm(mpg ~ carb))) +# Adding some columns. Use ftransform<- to also replace existing ones +add_vars(iris) <- num_vars(iris) |> fsum(TRA = '%') |> add_stub("perc_") +@ +<>= +iris <- iris2 +@ +\vspace{-2mm} + + +\hrrule +\section{Multi-Type Aggregation} +\itxt{Convenient interface to complex multi-type aggregations}\\ \vspace{1mm} +\code{collap(data, by, FUN = fmean, catFUN = fmode, \\ +\quad\qquad\ cols = NULL, w = NULL, wFUN = fsum,\\ +\quad\qquad\ custom = NULL, keep.col.order = TRUE, ...)}\\ [0.3em] + +%\textbf{Example} \\ + +<<>>= +# Population weighted mean (PCGDP, LIFEEX) & mode (country), and sum(POP) +collap(wlddev, country + PCGDP + LIFEEX ~ income, w = ~ POP) +@ +%\vspace{-2mm} +\end{multicols} % \vspace{-20mm} +%\vspace{20mm} + +% \end{adjustbox} +%} +% \hrrule +\vspace{-5mm} +\textcolor{lightgray}{\hrulefill}\\ +{\scriptsize \vspace{-0.5mm} + Page 1 of 2 \hfill \href{https://creativecommons.org/licenses/by-sa/4.0/}{CC-BY-SA}\ Sebastian Krantz\ \textbullet\ Learn more at \href{https://sebkrantz.github.io/collapse/}{sebkrantz.github.io/collapse}\ \textbullet\ Source code at \href{https://github.com/SebKrantz/collapse}{github.com/SebKrantz/collapse}\ \textbullet\ Updates announced at \href{https://twitter.com/collapse\_R}{twitter.com/collapse\_R} - \#rcollapse\ \textbullet\ Cheatsheet created for \emph{collapse} version 2.0.3\ \textbullet\ Updated: 2023-10 +} + + + +\newpage + +% ------------------------------------------------------------------ +% Second Page +% ------------------------------------------------------------------ +\begin{multicols}{4} + +% \hrrule +\section{Advanced Transformations} +\itxt{Common transformations (in econometrics)} \\ [0.7em] +Scaling, Centering and Averaging\\ +\code{fscale(x, g = NULL, w = NULL, na.rm = TRUE,\\ +\hphantom{fscaleb}mean = 0, sd = 1, ...)}\\ +\code{fwithin(x, g = NULL, w = NULL, na.rm = TRUE, +\hphantom{fwithinb}mean = 0, theta = 1, ...)}\\ +\code{fbetween(x, g = NULL, w = NULL, na.rm = TRUE,\\ +\hphantom{fbetweenb}fill = FALSE, ...)} \\ [0.5em] + +Higher-Dimensional Centering/Avg. and Linear Prediction\\ +\code{fhdwithin(x, fl, w = NULL, na.rm = TRUE,\\ +\hphantom{fhdwithinb}fill = FALSE, lm.method = "qr", ...)}\\ +\code{fhdbetween()} - same arguments as \code{fhdwithin()} \\ [0.7em] + +Statistical Operators (function shorthands with extra features) \\ + + \code{STD(), W(), B(), HDW(), HDB()} \newline + +\textbf{Examples} + +<<>>= +# Grouped scaling +iris |> fgroup_by(Species) |> fscale() |> head(2) +STD(iris, ~ Species, stub = FALSE) |> invisible() # Same thing + faster +# Grouped and weighted scaling. Operators support formulas and keep ids +STD(mtcars, mpg + carb ~ cyl, w = ~ wt) |> head(2) +# Much shorter than fsubset(mpg > fmean(mpg, cyl, TRA = "replace")) +mtcars |> fsubset(mpg > B(mpg, cyl)) |> head(2) +# Regression with cyl fixed effects - a la Mundlak (1978) +lm(mpg ~ carb + B(carb, cyl), data = mtcars) |> coef() +# Fast grouped (vs) bivariate regression slopes: mpg ~ carb +mtcars |> fgroup_by(vs) |> fmutate(dm_carb = W(carb)) |> + fsummarise(beta = fsum(mpg, dm_carb) %/=% fsum(dm_carb^2)) +# Residuals from regressing on 'Petal' vars and 'Species' FE +fhdwithin(iris[1:2], iris[3:5]) |> head(2) +# Detrending with country-level cubic polynomials +HDW(wlddev, PCGDP + LIFEEX + POP ~ iso3c * poly(year, 3)) |> head(2) +# Note: HD centering/prediction and polynomials requires package 'fixest' +@ + % \emph{Note}: for higher-dimensional centerig / averaging, \emph{collapse} imports \emph{fixest}'s C++ demeaning algorithm, if available. +% \vspace{-2mm} + +\hrrule +\section{Linear Models} +\vspace{1mm} +Fast (barebones) linear model fitting with 6 different solvers\\ +\code{flm(y, X, w = NULL, add.icpt = FALSE, method = "lm")}\\ [0.5em] +Fast $R^2$-based F-test of exclusion restrictions for lm's (with FE)\\ +\code{fFtest(y, exc, X = NULL, w = NULL, full.df = TRUE)}\newline + +Both functions also have formula interfaces: +<<>>= +flm(cbind(mpg, disp) ~ hp + carb, weights = wt, mtcars) +# Test the exclusion of cyl-dummies and hp. +fFtest(mpg ~ qF(cyl) + hp | carb + qF(am), weights = wt, mtcars) +@ + +<>= +wlddev2 <- data.table::copy(wlddev) +vlabels(wlddev) <- NULL +options(width = 70) +@ + +% \hrrule +\section{Time Series and Panel Series} +\itxt{Fast and flexible indexed series and data frames: a modern upgrade of \emph{plm}'s 'pseries' and 'pdata.frame'}\newline + +Turn DF into an 'indexed\_frame' using id and/or time vars\\ +\code{data\_ix = findex\_by(data, id1, ..., time)}\\ [0.5em] +\code{data\_ix\$indexed\_series} - columns are 'indexed\_series' \\ [0.5em] +\code{index\_df = findex(data\_ix)} - retrieve 'index\_df': DF of ids \\ [0.5em] +\code{index\_df = with(data\_ix, findex(indexed\_series))} - can fetch 'index\_df' from 'indexed\_series' in any caller environment \\ [0.5em] +\code{data = unindex(data\_ix)} - unindex (also 'indexed\_series') \\ [0.5em] +\code{reindex(data, index = index\_df)} - reindex / new pointers\\ [0.5em] % (e.g. after \code{readRDS})\\ +'indexed\_series' can be 1-or-2D atomic objects. Vectors / time series / matrices can also be indexed directly using: \\ +\code{reindex(vec/mat, index = vec/index\_df)}\\ [0.5em] +\code{is\_irregular()} - irregularity in any index[ed] obj. or time vec\newline + +\textbf{Example: Indexing Panel Data} \\ +% , drop.index.levels = "none" +<<>>= +wldi <- wlddev |> findex_by(iso3c, year) # Balanced: 216 countries +fsubset(wldi, 1:2, iso3c, year, PCGDP:POP) +# Index stats: [N. ids] | [N. periods (tot.N. periods: (max-min)/GCD)] +LIFEEXi = wldi$LIFEEX # Indexed series +str(LIFEEXi, strict.width = "cut") +LIFEEXi[1:7] # Subsetting indexed series +c(is_irregular(LIFEEXi), is_irregular(LIFEEXi[-5])) # Is irregular? +@ +\emph{Note}: 'indexed\_series' and frames are supported via existing 'pseries'/'pdata.frame' methods for time series/panel functions. \newline + +% \vspace{2mm} +\itxt{Fast functions to perform time-based computations on (irregular) time series and (unbalanced) panel data}\newline + +Lags/Leads, Differences, Growth Rates and Cumulative Sums\\ +\code{flag(x, n = 1, g = NULL, t = NULL, fill = NA, ...)}\\ +\code{fdiff(x, n = 1, diff = 1, g = NULL, t = NULL,\\ +\hphantom{fdiffd}fill = NA, log = FALSE, rho = 1, ...)}\\ +\code{fgrowth(x, n = 1, diff = 1, g = NULL, t = NULL, fill\\ \ = NA, logdiff = FALSE, scale = 100, power = 1, ...)}\\ +\code{fcumsum(x, g = NULL, o = NULL, na.rm = TRUE,\\ +\hphantom{fcumsumd}fill = FALSE, check.o = TRUE, ...)}\\ [0.7em] + +Statistical Operators: \code{L(), F(), D(), Dlog(), G()} \newline + +\textbf{Example: Computing Growth Rates} \\ +<>= +# options(width = 70) +wldi = reindex(wldi) +@ +<<>>= +# Ad-hoc use: note that G() supports formulas which fgrowth() doesn't +fgrowth(AirPassengers) |> head() +G(wlddev, c(1, 10), by = PCGDP ~ iso3c, t = ~ year) |> ss(11:12) +wlddev |> fgroup_by(iso3c) |> fselect(iso3c, year, PCGDP, LIFEEX) |> + fmutate(PCGDP_growth = fgrowth(PCGDP, t = year)) |> head(2) +settransform(wlddev, PCGDP_growth = G(PCGDP, g = iso3c, t = year)) +# Note: can omit t -> requires consecutive observations and groups +# Usage with indexed series / frames: +@ +<<>>= +G(wldi) |> head(2) # default: compute growth of num_vars(), keep ids +settransform(wldi, PCGDP_growth = fgrowth(PCGDP)) +lm(G(PCGDP) ~ L(G(LIFEEX), 0:2), wldi) |> summary() |> coef() |> round(3) +@ + +% lm(G(PCGDP) ~ L(G(LIFEEX), 0:3), wldi) |> summary() |> coef() |> round(3) +% flag(vec/mat/DF, n = -1:3, g = id, t = time) +% L(DF, n = -1:2, by = ~ id1 + id2, t = ~ time) +% Use with indexed data or grouped data: +% data_ix |> L(); data_ix |> fmutate(L_var = L(var)) +% lm(G(var1) ~ L(G(var2), 0:2), data_ix) +% data |> fgroup_by(id) |> flag(t = time) +% data |> gby(id) |> fmutate(Lv = L(var, t = time)) +% +% Cumulative Sums: +% fcumsum(x, g = NULL, o = NULL, fill = FALSE, ) +% \vspace{2mm} +\code{psacf(), pspacf(), psccf()} - panel series ACF/PACF/CCF\\ [0.3em] +\code{psmat()} - panel data to array conversion/reshaping\\ % \newpage +% \vspace{-5mm} +% <>= +% psacf(G(LIFEEXi), main = NA) # ACF of growth rate of life expectancy +% @ +% +% <>= +% psmat(LIFEEXi) |> plot(colours = TRUE) +% @ +% +% \vspace{-1mm} +\hrrule +\section{Summary Statistics} +\code{qsu()} - fast (grouped, weighted, panel-decomposed)\\ +\qquad summary statistics for cross-sectional and panel data\\ +<<>>= +# Panel data statistics: overall, on group-means and group-centered data +qsu(iris, pid = Sepal.Length ~ Species, higher = TRUE) +@ +\code{qtab()} - faster \code{table()} function, incl. weights \& custom funs\\ [0.5em] +\code{descr()} - detailed statistical description of data.frame\\ [0.5em] +\code{varying()} - check variation within groups (panel-ids)\\ [0.5em] +\code{pwcor(), pwcov(), pwnobs()} - pairwise correlations,\\ \qquad covariance and obs. (with P-value and pretty printing) +% \vspace{-1mm} + +\hrrule +\section{List Processing} +%\vspace{1mm} +\itxt{Functions to process (nested) lists (of data objects)}\\ [0.5em] +%\setstretch{1.5} +\code{ldepth()} - level of nesting of list\\ [0.5em] +\code{is\_unlistable()} - is list composed of atomic objects\\ [0.5em] +\code{has\_elem()} - search if list contains certain elements\\ [0.5em] +\code{get\_elem()} - pull out elements from list / subset list\\ [0.5em] +\code{atomic\_elem[<-](), list\_elem[<-]()} - get list with atomic /\\ \qquad sub-list elements, examining only first level of list\\ [0.3em] +\code{reg\_elem(), irreg\_elem()} - get full list tree leading to atomic\\ \qquad ('regular') or non-atomic ('irregular') elements\\ [0.5em] +\code{rsplit()} - efficient (recursive) splitting\\ [0.5em] +\code{t\_list()} - efficient list transpose (transpose lists of lists)\\ [0.5em] +\code{rapply2d()} - recursive apply to lists of data objects\\ [0.5em] +\code{unlist2d()} - recursive row-binding to data.frame\\ [0.5em] +%\setstretch{1} + +\textbf{Example: Nested Linear Models} \\ %\vspace{-1mm} +<<>>= +(dl <- mtcars |> rsplit(mpg + hp + carb ~ vs + am)) |> str(max.level = 2) +nest_lm <- dl |> rapply2d(lm, formula = mpg ~ .) +(nest_coef <- nest_lm |> rapply2d(summary, classes = "lm") |> + get_elem("coefficients")) |> str(give.attr = FALSE, strict = "cut") +nest_coef |> unlist2d(c("vs", "am"), row.names = "variable") |> head(2) +@ +% \vspace{-2mm} +% \columnbreak + +% \hrrule +\section{Recode and Replace Values} +\code{recode\_num(), recode\_char()} - recode numeric / character values (+ regex recoding) in matrix-like objects\\ [0.5em] +\code{replace\_[NA|Inf|outliers]()} - replace special values\\ [0.5em] +\code{pad()} - add (missing) observations / rows i.e. expand objects +% \vspace{-1mm} + +\hrrule +\section{(Memory) Efficient Programming} +\itxt{Functions for (memory) efficient R programming}\\ [0.5em] + +\code{any|all[v|NA]}, \code{which[v|NA]}, \code{\%[=|!]=\%}, \code{copyv}, \code{setv}, \code{alloc} \code{missing\_cases}, \code{na\_[insert|rm|omit]}, \code{vlengths}, \code{vtypes}, \code{vgcd}, \code{fnlevels}, \code{fn[row|col]}, \code{fdim}, \code{seq\_[row|col]}, \code{vec}\\ +<>= +fsubset(wlddev, year %==% 2010) # 2x faster fsubset(wlddev, year == 2010) +attach(mtcars) # Efficient sub-assignment by reference, various options... +setv(am, 0, vs); setv(am, 1:10, vs); setv(am, 1:10, vs[10:20]) +@ +% x <- na_insert(rnorm(1e7)) +% microbenchmark(na_rm(x), x[!is.na(x)]) +% microbenchmark(na.omit(wlddev), na_omit(wlddev, na.attr = TRUE)) +% # Efficient subassignment by reference +% with(wlddev, setv(POP, year %==% 2010, fmax(POP))) +% microbenchmark(A = fsubset(wlddev, year == 2010), B = fsubset(wlddev, year %==% 2010)) +% x <- rnorm(1e7); xNA <- na_insert(x) +% microbenchmark(setv(xNA, NA, x), ) +% with(wlddev, setv(POP, year %==% 2010, fmax(POP))) +% \vspace{-1mm} + +\hrrule +\section{Small (Helper) Functions} +\itxt{Functions for (meta-)programming and attributes}\\ [0.5em] +\code{.c}, \code{massign}, \code{\%=\%}, \code{vlabels[<-]}, \code{setLabels}, \code{vclasses}, \code{namlab}, \code{[add|rm]\_stub}, \code{all\_identical}, \code{all\_obj\_equal}, \code{all\_funs}, \code{set[Dim|Row|Col]names}, \code{unattrib}, \code{setAttrib}, \code{copyAttrib}, \code{copyMostAttrib}, \code{is\_categorical}, \code{is\_date} + +<>= +wlddev <- wlddev2 +@ +<<>>= +.c(var1, var2, var3) # Non-standard concatenation +.c(values, vectors) %=% eigen(cov(mtcars)) # Multiple Assignment +# Variable labels: vlabels[<-], [set]relabel() etc. namlab() shows summary +namlab(wlddev[c(2, 9)], N = TRUE, Ndist = TRUE, class = TRUE) +@ +% \vspace{-1mm} + + +\hrrule +\section{API Extensions and Global Options} +\itxt{Shorthands for frequently used functions}\\ [0.5em] +\code{fselect -> slt, fsubset -> sbt, fmutate -> mtt, [f/set]transform[v] -> [set]tfm[v], fsummarise -> smr, +across -> acr, fgroup\_by -> gby, finteraction -> itn, findex\_by -> iby, findex -> ix, frename -> rnm, get\_vars -> gv, num\_vars -> nv, +add\_vars -> av} \newline + +\itxt{Namespace masking and other global options}\\ [0.5em] +Use \code{set\_collpse(mask = c(...))} with a vector of functions starting with f-, to export versions without f-, masking base R and/or \emph{dplyr}. A few keywords exist to mask multiple functions, see \code{help("collapse-options")}. There are also many other global defaults and optimizations that can be controlled with \code{set\_collapse(...)}. Retrieve options using \code{get\_collapse()}. + +<>= +# Masking all (f-)functions and changing some defaults (=optimizing) +library(collapse) +set_collapse(mask = "all", na.rm = FALSE, sort = FALSE, nthreads = 4) +# The following is now 100% collapse code and executed without regard for +# missing values, using unsorted grouping and 4 threads (where applicable) +wlddev |> + subset(year >= 1990 & is.finite(GINI)) |> + group_by(year) |> + summarise(n = n(), across(PCGDP:GINI, mean, w = POP)) + +with(mtcars, table(cyl, vs, am)) +sum(mtcars) +diff(EuStockMarkets) +droplevels(wlddev) +mean(nv(iris), g = iris$Species) +scale(nv(GGDC10S), g = GGDC10S$Variable) +unique(GGDC10S, cols = c("Variable", "Country")) +range(wlddev$date) + +wlddev |> + index_by(iso3c, year) |> + mutate(PCGDP_lag = lag(PCGDP), + PCGDP_diff = PCGDP - PCGDP_lag, + PCGDP_growth = growth(PCGDP)) |> unindex() + +@ + + +\end{multicols} + +\vspace{-5.5mm} +\textcolor{lightgray}{\hrulefill}\\ +{\scriptsize \vspace{-0.5mm} + Page 2 of 2 \hfill \href{https://creativecommons.org/licenses/by-sa/4.0/}{CC-BY-SA}\ Sebastian Krantz\ \textbullet\ Learn more at \href{https://sebkrantz.github.io/collapse/}{sebkrantz.github.io/collapse}\ \textbullet\ Source code at \href{https://github.com/SebKrantz/collapse}{github.com/SebKrantz/collapse}\ \textbullet\ Updates announced at \href{https://twitter.com/collapse\_R}{twitter.com/collapse\_R} - \#rcollapse\ \textbullet\ Cheatsheet created for \emph{collapse} version 2.0.3\ \textbullet\ Updated: 2023-10 +} + +\end{document} diff --git a/latex/collapse/collapse_cheat_sheet.pdf b/latex/collapse/collapse_cheat_sheet.pdf new file mode 100644 index 00000000..e83f05ae Binary files /dev/null and b/latex/collapse/collapse_cheat_sheet.pdf differ diff --git a/latex/collapse/collapse_logo_vsmall.png b/latex/collapse/collapse_logo_vsmall.png new file mode 100644 index 00000000..0eb0a0c7 Binary files /dev/null and b/latex/collapse/collapse_logo_vsmall.png differ diff --git a/pngs/collapse.png b/pngs/collapse.png index a52f2412..7646c7ca 100644 Binary files a/pngs/collapse.png and b/pngs/collapse.png differ diff --git a/powerpoints/collapse.pptx b/powerpoints/collapse.pptx deleted file mode 100644 index 6a7f0133..00000000 Binary files a/powerpoints/collapse.pptx and /dev/null differ