Improve type stability of dynamic branching (#1106)

Improve type stability of dynamic branching

DESCRIPTION

@@ -111,7 +111,8 @@ Imports:
   rlang (>= 0.2.0),
   storr (>= 1.1.0),
   txtq (>= 0.1.3),
-  utils
+  utils,
+  vctrs (>= 0.2.0)
 Suggests:
   abind,
   bindr,

NAMESPACE

@@ -250,4 +250,6 @@ importFrom(utils,stack)
 importFrom(utils,type.convert)
 importFrom(utils,unzip)
 importFrom(utils,write.table)
+importFrom(vctrs,vec_c)
+importFrom(vctrs,vec_slice)
 useDynLib(drake, .registration = TRUE)

NEWS.md

@@ -1,5 +1,9 @@
 # Version 7.8.0.9000
 
+## Breaking changes
+
+- Embrace the `vctrs` paradigm and its type stability for dynamic branching (#1105, #1106).
+
 ## New features
 
 - Add a new `log_build_times` argument to `make()` and `drake_config()`. Allows users to disable the recording of build times. Produces a speedup of up to 20% on Macs (#1078).

R/cache.R

@@ -323,7 +323,8 @@ get_subtargets.drake_dynamic <- function(hashes, cache, subtargets) {
   if (!is.null(subtargets)) {
     hashes <- hashes[subtargets]
   }
-  lapply(hashes, cache$get_value, use_cache = FALSE)
+  out <- lapply(hashes, cache$get_value, use_cache = FALSE)
+  do.call(vec_c, out)
 }
 
 get_subtargets.default <- function(hashes, cache, subtargets) {

R/drake_plan.R

@@ -253,18 +253,18 @@
 #' drake_plan(x = target(f(char), transform = map(char = !!sms)))
 #'
 #' # Dynamic branching
+#' # Get the mean mpg for each cyl in the mtcars dataset.
 #' plan <- drake_plan(
-#'   w = c("a", "a", "b", "b"),
-#'   x = seq_len(4),
-#'   y = target(x + 1, dynamic = map(x)),
-#'   z = target(list(y = y, w = w), dynamic = group(y, .by = w))
+#'   raw = mtcars,
+#'   group_index = raw$cyl,
+#'   munged = target(raw[, c("mpg", "cyl")], dynamic = map(raw)),
+#'   mean_mpg_by_cyl = target(
+#'     data.frame(mpg = mean(munged$mpg), cyl = munged$cyl[1]),
+#'     dynamic = group(munged, .by = group_index)
+#'   )
 #' )
 #' make(plan)
-#' subtargets(y)
-#' readd(subtargets(y)[1], character_only = TRUE)
-#' readd(subtargets(y)[2], character_only = TRUE)
-#' readd(subtargets(z)[1], character_only = TRUE)
-#' readd(subtargets(z)[2], character_only = TRUE)
+#' readd(mean_mpg_by_cyl)
 #' })
 #' }
 drake_plan <- function(

R/drake_plan_helpers.R

@@ -525,9 +525,7 @@ no_deps <- function(x = NULL) {
 #'   y = target(id_chr(), dynamic = map(x))
 #' )
 #' make(plan)
-#' ys <- subtargets(y)
-#' ys
-#' readd(ys[1], character_only = TRUE)
+#' readd(y, subtargets = 1)
 #' # Static branching
 #' plan <- drake_plan(
 #'   y = target(c(x, .id_chr), transform = map(x = !!seq_len(4)))

R/dynamic.R

@@ -13,14 +13,14 @@
 #'   w = c("a", "a", "b", "b"),
 #'   x = seq_len(4),
 #'   y = target(x + 1, dynamic = map(x)),
-#'   z = target(list(y = y, w = w), dynamic = group(y, .by = w))
+#'   z = target(sum(x) + sum(y), dynamic = group(x, y, .by = w))
 #' )
 #' make(plan)
 #' subtargets(y)
-#' readd(subtargets(y)[1], character_only = TRUE)
-#' readd(subtargets(y)[2], character_only = TRUE)
-#' readd(subtargets(z)[1], character_only = TRUE)
-#' readd(subtargets(z)[2], character_only = TRUE)
+#' subtargets(z)
+#' readd(x)
+#' readd(y)
+#' readd(z)
 #' })
 #' }
 subtargets <- function(
@@ -69,15 +69,18 @@ subtargets <- function(
 #'
 #'   # The first trace lets us see the values of w
 #'   # that go with the sub-targets of y.
-#'   y = target(c(w, x), dynamic = cross(w, x, .trace = w)),
+#'   y = target(paste0(w, x), dynamic = cross(w, x, .trace = w)),
 #'
 #'   # We can use the trace as a grouping variable for the next
 #'   # group().
 #'   w_tr = get_trace("w", y),
 #'
 #'   # Now, we use the trace again to keep track of the
 #'   # values of w corresponding to the sub-targets of z.
-#'   z = target(y, dynamic = group(y, .by = w_tr, .trace = w_tr))
+#'   z = target(
+#'     paste0(y, collapse = "-"),
+#'     dynamic = group(y, .by = w_tr, .trace = w_tr)
+#'   )
 #' )
 #' make(plan)
 #'
@@ -88,7 +91,7 @@ subtargets <- function(
 #' read_trace("w", "y")
 #'
 #' # And we know which values of `w_tr` (and thus `w`)
-#' # match up with the sub-targets of `z`.
+#' # match up with the sub-targets of `y`.
 #' readd(z)
 #' read_trace("w_tr", "z")
 #' })
@@ -131,20 +134,30 @@ read_trace <- function(
 #'
 #'   # The first trace lets us see the values of w
 #'   # that go with the sub-targets of y.
-#'   y = target(c(w, x), dynamic = cross(w, x, .trace = w)),
+#'   y = target(paste0(w, x), dynamic = cross(w, x, .trace = w)),
 #'
 #'   # We can use the trace as a grouping variable for the next
 #'   # group().
 #'   w_tr = get_trace("w", y),
 #'
 #'   # Now, we use the trace again to keep track of the
 #'   # values of w corresponding to the sub-targets of z.
-#'   z = target(y, dynamic = group(y, .by = w_tr, .trace = w_tr))
+#'   z = target(
+#'     paste0(y, collapse = "-"),
+#'     dynamic = group(y, .by = w_tr, .trace = w_tr)
+#'   )
 #' )
 #' make(plan)
 #'
 #' # We can read the trace outside make().
+#' # That way, we know which values of `w` correspond
+#' # to the sub-targets of `y`.
+#' readd(y)
 #' read_trace("w", "y")
+#'
+#' # And we know which values of `w_tr` (and thus `w`)
+#' # match up with the sub-targets of `y`.
+#' readd(z)
 #' read_trace("w_tr", "z")
 #' })
 #' }
@@ -412,36 +425,7 @@ as_dynamic <- function(x) {
 }
 
 dynamic_subvalue <- function(value, index) {
-  UseMethod("dynamic_subvalue")
-}
-
-dynamic_subvalue.data.frame <- function(value, index) {
-  value[index,, drop = FALSE] # nolint
-}
-
-dynamic_subvalue.drake_dynamic <- function(value, index) {
-  dynamic_subvalue_vector(value, index)
-}
-
-dynamic_subvalue.default <- function(value, index) {
-  if (is.null(dim(value))) {
-    dynamic_subvalue_vector(value, index)
-  } else {
-    dynamic_subvalue_array(value, index)
-  }
-}
-
-dynamic_subvalue_array <- function(value, index) {
-  ref <- slice.index(value, 1L)
-  out <- value[ref %in% index]
-  dim <- dim(value)
-  dim[1] <- length(index)
-  dim(out) <- dim
-  out
-}
-
-dynamic_subvalue_vector <- function(value, index) {
-  value[index]
+  vec_slice(x = value, i = index)
 }
 
 match_dynamic_call <- function(dynamic) {

R/make.R

@@ -109,18 +109,18 @@
 #' clean() # Start from scratch next time around.
 #' }
 #' # Dynamic branching
+#' # Get the mean mpg for each cyl in the mtcars dataset.
 #' plan <- drake_plan(
-#'   w = c("a", "a", "b", "b"),
-#'   x = seq_len(4),
-#'   y = target(x + 1, dynamic = map(x)),
-#'   z = target(list(y = y, w = w), dynamic = group(y, .by = w))
+#'   raw = mtcars,
+#'   group_index = raw$cyl,
+#'   munged = target(raw[, c("mpg", "cyl")], dynamic = map(raw)),
+#'   mean_mpg_by_cyl = target(
+#'     data.frame(mpg = mean(munged$mpg), cyl = munged$cyl[1]),
+#'     dynamic = group(munged, .by = group_index)
+#'   )
 #' )
 #' make(plan)
-#' subtargets(y)
-#' readd(subtargets(y)[1], character_only = TRUE)
-#' readd(subtargets(y)[2], character_only = TRUE)
-#' readd(subtargets(z)[1], character_only = TRUE)
-#' readd(subtargets(z)[2], character_only = TRUE)
+#' readd(mean_mpg_by_cyl)
 #' })
 #' }
 make <- function(

R/manage_memory.R

@@ -17,6 +17,9 @@ manage_memory <- function(target, config, downstream = NULL, jobs = 1) {
     memory_strategy <- config$memory_strategy
   }
   class(target) <- memory_strategy
+  if (!is_subtarget(target, config)) {
+    clear_envir_subtargets(target = target, config = config)
+  }
   manage_deps(
     target = target,
     config = config,
@@ -47,7 +50,6 @@ manage_deps.autoclean <- function(target, config, downstream, jobs) {
   target_deps <- deps_memory(targets = target, config = config)
   discard_these <- setdiff(x = already_loaded, y = target_deps)
   discard_targets(discard_these, target, config)
-  clear_envir_subtargets(target = target, config = config)
   target_deps <- setdiff(target_deps, target)
   target_deps <- setdiff(target_deps, already_loaded)
   try_load(targets = target_deps, config = config, jobs = jobs)
@@ -86,7 +88,6 @@ discard_targets <- function(discard_these, target, config) {
 }
 
 manage_deps.unload <- function(target, config, downstream, jobs) {
-  clear_envir_subtargets(target = target, config = config)
   clear_envir_targets(target = target, config = config)
 }
 
@@ -95,7 +96,6 @@ manage_deps.none <- function(target, config, downstream, jobs) {
 }
 
 clear_envir_subtargets <- function(target, config) {
-  config$logger$minor("clear subtarget envir", target = target)
   rm(list = config$envir_loaded$subtargets, envir = config$envir_subtargets)
   config$envir_loaded$subtargets <- character(0)
   config$envir_subtargets[[drake_envir_marker]] <- TRUE
@@ -280,6 +280,7 @@ load_dynamic_subdep_impl.group <- function( # nolint
 ) {
   subdeps <- config$cache$get(dep, namespace = "meta")$subtargets[index]
   value <- config$cache$mget(subdeps, use_cache = FALSE)
+  value <- do.call(vec_c, value)
   assign(
     x = dep,
     value = value,

R/package.R

@@ -29,18 +29,18 @@
 #' head(large)
 #' }
 #' # Dynamic branching
+#' # Get the mean mpg for each cyl in the mtcars dataset.
 #' plan <- drake_plan(
-#'   w = c("a", "a", "b", "b"),
-#'   x = seq_len(4),
-#'   y = target(x + 1, dynamic = map(x)),
-#'   z = target(list(y = y, w = w), dynamic = group(y, .by = w))
+#'   raw = mtcars,
+#'   group_index = raw$cyl,
+#'   munged = target(raw[, c("mpg", "cyl")], dynamic = map(raw)),
+#'   mean_mpg_by_cyl = target(
+#'     data.frame(mpg = mean(munged$mpg), cyl = munged$cyl[1]),
+#'     dynamic = group(munged, .by = group_index)
+#'   )
 #' )
 #' make(plan)
-#' subtargets(y)
-#' readd(subtargets(y)[1], character_only = TRUE)
-#' readd(subtargets(y)[2], character_only = TRUE)
-#' readd(subtargets(z)[1], character_only = TRUE)
-#' readd(subtargets(z)[2], character_only = TRUE)
+#' readd(mean_mpg_by_cyl)
 #' })
 #' }
 #' @references <https://github.com/ropensci/drake>
@@ -57,4 +57,5 @@
 #' @importFrom txtq txtq
 #' @importFrom utils compareVersion flush.console head menu packageVersion
 #'   read.csv sessionInfo stack type.convert unzip write.table
+#' @importFrom vctrs vec_c vec_slice
 NULL