From 322785ead30be24da59ed8f19fbebb26c43c3803 Mon Sep 17 00:00:00 2001 From: wlandau Date: Sun, 15 Dec 2019 20:14:19 -0500 Subject: [PATCH] Change the name "layout" to "spec", e.g. config$spec --- NEWS.md | 1 + R/backend_clustermq.R | 20 +-- R/backend_future.R | 16 +-- R/cache.R | 4 +- R/create_drake_graph.R | 10 +- ...ate_drake_layout.R => create_drake_spec.R} | 126 +++++++++--------- R/deps.R | 10 +- R/drake_build.R | 4 +- R/drake_config.R | 64 ++++----- R/drake_graph_info.R | 6 +- R/drake_meta_.R | 36 ++--- R/dynamic.R | 90 ++++++------- R/handle_triggers.R | 4 +- R/hpc.R | 40 +++--- R/local_build.R | 18 +-- R/make.R | 2 + R/manage_memory.R | 20 +-- R/store_outputs.R | 8 +- man/cached_planned.Rd | 17 +++ man/cached_unplanned.Rd | 20 +++ man/cmq_build.Rd | 4 +- man/drake_config.Rd | 12 +- man/future_build.Rd | 2 +- man/make.Rd | 12 +- man/tracked.Rd | 2 +- tests/testthat/test-3-commands.R | 14 +- tests/testthat/test-6-hpc.R | 2 +- tests/testthat/test-6-plans.R | 4 +- tests/testthat/test-6-triggers.R | 10 +- tests/testthat/test-7-deprecate.R | 2 +- tests/testthat/test-8-cache.R | 2 +- tests/testthat/test-9-dynamic.R | 24 ++-- 32 files changed, 327 insertions(+), 279 deletions(-) rename R/{create_drake_layout.R => create_drake_spec.R} (76%) diff --git a/NEWS.md b/NEWS.md index 3953383ef..c1de889cf 100644 --- a/NEWS.md +++ b/NEWS.md @@ -33,6 +33,7 @@ - Write a complete project structure in `use_drake()` (#1097, @lorenzwalthert, @tjmahr). - Add a minor logger note to say how many dynamic sub-targets are registered at a time (#1102, @kendonB). - Handle dependencies that are dynamic targets but not declared as such for the current target (#1107). +- Internally, the "layout" data structure is now called the "workflow specification", or "spec" for short. The spec is `drake`'s interpretation of the plan. In the plan, all the dependency relationships among targets and files are *implicit*. In the spec, they are all *explicit*. We get from the plan to the spec using static code analysis, e.g. `analyze_code()`. # Version 7.8.0 diff --git a/R/backend_clustermq.R b/R/backend_clustermq.R index b90c3b5cb..f31da07ef 100644 --- a/R/backend_clustermq.R +++ b/R/backend_clustermq.R @@ -140,14 +140,14 @@ cmq_send_target <- function(target, config) { manage_memory(target = target, config = config, jobs = 1) deps <- cmq_deps_list(target, config) } - layout <- hpc_layout(target, config) + spec <- hpc_spec(target, config) ht_is_subtarget <- config$ht_is_subtarget config$workers$send_call( expr = drake::cmq_build( target = target, meta = meta, deps = deps, - layout = layout, + spec = spec, ht_is_subtarget = ht_is_subtarget, config = config ), @@ -155,17 +155,17 @@ cmq_send_target <- function(target, config) { target = target, meta = meta, deps = deps, - layout = layout, + spec = spec, ht_is_subtarget = ht_is_subtarget ) ) } cmq_deps_list <- function(target, config) { - layout <- config$layout[[target]] - keys_static <- layout$deps_build$memory - keys_dynamic <- layout$deps_dynamic_whole - keys_subtargets <- layout$deps_dynamic + spec <- config$spec[[target]] + keys_static <- spec$deps_build$memory + keys_dynamic <- spec$deps_dynamic_whole + keys_subtargets <- spec$deps_dynamic vals_static <- lapply( keys_static, get, @@ -202,12 +202,12 @@ cmq_deps_list <- function(target, config) { #' @param target Target name. #' @param meta List of metadata. #' @param deps Named list of target dependencies. -#' @param layout Internal, part of the full `config$layout`. +#' @param spec Internal, part of the full `config$spec`. #' @param ht_is_subtarget Internal, part of `config` #' @param config A [drake_config()] list. -cmq_build <- function(target, meta, deps, layout, ht_is_subtarget, config) { +cmq_build <- function(target, meta, deps, spec, ht_is_subtarget, config) { config$logger$minor("build on an hpc worker", target = target) - config$layout <- layout + config$spec <- spec config$ht_is_subtarget <- ht_is_subtarget do_prework(config = config, verbose_packages = FALSE) caching <- hpc_caching(target, config) diff --git a/R/backend_future.R b/R/backend_future.R index 826d5e5f2..370ad78bd 100644 --- a/R/backend_future.R +++ b/R/backend_future.R @@ -82,12 +82,12 @@ ft_launch_worker <- function(target, meta, protect, config) { manage_memory(target = target, config = config, downstream = protect) } DRAKE_GLOBALS__ <- NULL # Avoid name conflicts with other globals. - layout <- hpc_layout(target, config) + spec <- hpc_spec(target, config) globals <- future_globals( target = target, meta = meta, config = config$ft_config, - layout = layout, + spec = spec, ht_is_subtarget = config$ht_is_subtarget, protect = protect ) @@ -98,13 +98,13 @@ ft_launch_worker <- function(target, meta, protect, config) { target = DRAKE_GLOBALS__$target, meta = DRAKE_GLOBALS__$meta, config = DRAKE_GLOBALS__$config, - layout = DRAKE_GLOBALS__$layout, + spec = DRAKE_GLOBALS__$spec, ht_is_subtarget = DRAKE_GLOBALS__$ht_is_subtarget, protect = DRAKE_GLOBALS__$protect ), globals = globals, label = target, - resources = as.list(layout$resources) + resources = as.list(spec$resources) ), target = target ) @@ -114,7 +114,7 @@ future_globals <- function( target, meta, config, - layout, + spec, ht_is_subtarget, protect ) { @@ -123,7 +123,7 @@ future_globals <- function( target = target, meta = meta, config = config, - layout = layout, + spec = spec, ht_is_subtarget = ht_is_subtarget, protect = protect ) @@ -161,11 +161,11 @@ future_build <- function( target, meta, config, - layout, + spec, ht_is_subtarget, protect ) { - config$layout <- layout + config$spec <- spec config$ht_is_subtarget <- ht_is_subtarget caching <- hpc_caching(target, config) if (identical(caching, "worker")) { diff --git a/R/cache.R b/R/cache.R index 096c499e3..97819bc62 100644 --- a/R/cache.R +++ b/R/cache.R @@ -617,6 +617,7 @@ cached <- function( #' @param plan A drake plan. #' @examples #' \dontrun{ +#' isolate_example("cache_planned() example", { #' plan <- drake_plan(w = 1) #' make(plan) #' cached_planned(plan) @@ -628,7 +629,6 @@ cached <- function( #' make(plan) #' cached_planned(plan) #' cached() -#' } #' }) #' } cached_planned <- function( @@ -661,6 +661,7 @@ cached_planned <- function( #' @param plan A drake plan. #' @examples #' \dontrun{ +#' isolate_example("cache_unplanned() example", { #' plan <- drake_plan(w = 1) #' make(plan) #' cached_unplanned(plan) @@ -675,7 +676,6 @@ cached_planned <- function( #' cached() #' clean(list = cached_unplanned(plan)) #' cached() -#' } #' }) #' } cached_unplanned <- function( diff --git a/R/create_drake_graph.R b/R/create_drake_graph.R index 38a5a17f1..f84d29bbe 100644 --- a/R/create_drake_graph.R +++ b/R/create_drake_graph.R @@ -1,6 +1,6 @@ create_drake_graph <- function( plan, - layout, + spec, targets, cache, jobs, @@ -8,10 +8,10 @@ create_drake_graph <- function( ) { config <- list(plan = plan, jobs = jobs, logger = logger, cache = cache) edges <- memo_expr( - cdg_create_edges(config, layout), + cdg_create_edges(config, spec), cache, plan, - layout + spec ) memo_expr( cdg_finalize_graph(edges, targets, config), @@ -21,9 +21,9 @@ create_drake_graph <- function( ) } -cdg_create_edges <- function(config, layout) { +cdg_create_edges <- function(config, spec) { edges <- lightly_parallelize( - X = layout, + X = spec, FUN = cdg_node_to_edges, jobs = config$jobs, config = config diff --git a/R/create_drake_layout.R b/R/create_drake_spec.R similarity index 76% rename from R/create_drake_layout.R rename to R/create_drake_spec.R index 502e23e3a..b6d30ecc6 100644 --- a/R/create_drake_layout.R +++ b/R/create_drake_spec.R @@ -1,4 +1,4 @@ -create_drake_layout <- function( +create_drake_spec <- function( plan, envir = parent.frame(), logger, @@ -21,31 +21,31 @@ create_drake_layout <- function( ) imports <- cdl_prepare_imports(config) imports_kernel <- cdl_imports_kernel(config, imports) - import_layout <- memo_expr( + import_spec <- memo_expr( cdl_analyze_imports(config, imports), config$cache, imports_kernel ) knitr_hash <- cdl_get_knitr_hash(config) - command_layout <- memo_expr( + command_spec <- memo_expr( cdl_analyze_commands(config), config$cache, config$plan, config$trigger, - import_layout, + import_spec, imports_kernel, knitr_hash ) - cdl_set_knitr_files(config = config, layout = command_layout) - out <- c(import_layout, command_layout) + cdl_set_knitr_files(config = config, spec = command_spec) + out <- c(import_spec, command_spec) list2env(out, parent = emptyenv(), hash = TRUE) } # https://github.com/ropensci/drake/issues/887 # nolint -cdl_set_knitr_files <- function(config, layout) { +cdl_set_knitr_files <- function(config, spec) { config$logger$minor("set knitr files") knitr_files <- lightly_parallelize( - X = layout, + X = spec, FUN = function(x) { x$deps_build$knitr_in }, @@ -59,7 +59,7 @@ cdl_set_knitr_files <- function(config, layout) { ) } -cdl_get_knitr_hash <- function(config, layout) { +cdl_get_knitr_hash <- function(config, spec) { config$logger$minor("get knitr hash") if (!config$cache$exists(key = "knitr", namespace = "memoize")) { out <- config$cache$digest("", serialize = FALSE) @@ -148,8 +148,8 @@ cdl_analyze_commands <- function(config) { envir = config$envir_targets ) } - layout <- drake_pmap(.l = config$plan, .f = list, jobs = config$jobs) - names(layout) <- config$plan$target + spec <- drake_pmap(.l = config$plan, .f = list, jobs = config$jobs) + names(spec) <- config$plan$target config$default_condition_deps <- cdl_import_dependencies( config$trigger$condition, allowed_globals = config$ht_globals @@ -159,8 +159,8 @@ cdl_analyze_commands <- function(config) { allowed_globals = config$ht_globals ) out <- lightly_parallelize( - X = layout, - FUN = cdl_prepare_layout, + X = spec, + FUN = cdl_prepare_spec, jobs = config$jobs, config = config ) @@ -168,74 +168,74 @@ cdl_analyze_commands <- function(config) { out } -cdl_prepare_layout <- function(config, layout) { - target <- layout$target +cdl_prepare_spec <- function(config, spec) { + target <- spec$target config$logger$minor("analyze", target = target) - layout$dynamic <- as_dynamic(layout$dynamic) - layout$deps_build <- cdl_command_dependencies( - command = layout$command, - exclude = layout$target, + spec$dynamic <- as_dynamic(spec$dynamic) + spec$deps_build <- cdl_command_dependencies( + command = spec$command, + exclude = spec$target, allowed_globals = config$ht_globals ) - layout$deps_dynamic <- cdl_dynamic_deps( - layout$dynamic, - layout$target, + spec$deps_dynamic <- cdl_dynamic_deps( + spec$dynamic, + spec$target, config ) - layout$deps_dynamic_trace <- cdl_dynamic_trace(layout$dynamic, config) - cdl_assert_trace(layout$dynamic, layout) - layout$command_standardized <- cdl_standardize_command(layout$command) - if (inherits(layout$dynamic, "dynamic")) { - dynamic_command <- cdl_std_dyn_cmd(layout$dynamic) - layout$command_standardized <- paste( - layout$command_standardized, + spec$deps_dynamic_trace <- cdl_dynamic_trace(spec$dynamic, config) + cdl_assert_trace(spec$dynamic, spec) + spec$command_standardized <- cdl_standardize_command(spec$command) + if (inherits(spec$dynamic, "dynamic")) { + dynamic_command <- cdl_std_dyn_cmd(spec$dynamic) + spec$command_standardized <- paste( + spec$command_standardized, dynamic_command ) } - layout$command_build <- cdl_preprocess_command( - layout$command, + spec$command_build <- cdl_preprocess_command( + spec$command, config = config ) - if (is.null(layout$trigger) || all(is.na(layout$trigger))) { - layout$trigger <- config$trigger - layout$deps_condition <- config$default_condition_deps - layout$deps_change <- config$default_change_deps + if (is.null(spec$trigger) || all(is.na(spec$trigger))) { + spec$trigger <- config$trigger + spec$deps_condition <- config$default_condition_deps + spec$deps_change <- config$default_change_deps } else { - layout$deps_condition <- cdl_import_dependencies( - layout$trigger$condition, - exclude = layout$target, + spec$deps_condition <- cdl_import_dependencies( + spec$trigger$condition, + exclude = spec$target, allowed_globals = config$ht_globals ) - layout$deps_change <- cdl_import_dependencies( - layout$trigger$change, - exclude = layout$target, + spec$deps_change <- cdl_import_dependencies( + spec$trigger$change, + exclude = spec$target, allowed_globals = config$ht_globals ) } - cdl_no_dynamic_triggers(layout) + cdl_no_dynamic_triggers(spec) for (field in c("deps_build", "deps_condition", "deps_change")) { - layout[[field]]$memory <- ht_filter( + spec[[field]]$memory <- ht_filter( ht = config$ht_targets, - x = layout[[field]]$globals + x = spec[[field]]$globals ) } - layout$deps_build$memory <- base::union( - layout$deps_build$memory, - layout$deps_dynamic_trace + spec$deps_build$memory <- base::union( + spec$deps_build$memory, + spec$deps_dynamic_trace ) - layout + spec } -cdl_no_dynamic_triggers <- function(layout) { +cdl_no_dynamic_triggers <- function(spec) { cdl_no_dynamic_triggers_impl( - layout$target, - layout$deps_dynamic, - unlist(layout$deps_condition) + spec$target, + spec$deps_dynamic, + unlist(spec$deps_condition) ) cdl_no_dynamic_triggers_impl( - layout$target, - layout$deps_dynamic, - unlist(layout$deps_change) + spec$target, + spec$deps_dynamic, + unlist(spec$deps_change) ) } @@ -314,12 +314,12 @@ cdl_dynamic_trace.default <- function(dynamic, config) { character(0) } -cdl_assert_trace <- function(dynamic, layout) { +cdl_assert_trace <- function(dynamic, spec) { UseMethod("cdl_assert_trace") } -cdl_assert_trace.group <- function(dynamic, layout) { - bad <- setdiff(layout$deps_dynamic_trace, layout$deps_dynamic) +cdl_assert_trace.group <- function(dynamic, spec) { + bad <- setdiff(spec$deps_dynamic_trace, spec$deps_dynamic) if (!length(bad)) { return() } @@ -328,15 +328,15 @@ cdl_assert_trace.group <- function(dynamic, layout) { "the only legal dynamic trace variable ", "is the one you select with `.by`. ", "illegal dynamic trace variables for target ", - layout$target, + spec$target, ":\n", multiline_message(bad), call. = FALSE ) } -cdl_assert_trace.dynamic <- function(dynamic, layout) { - bad <- setdiff(layout$deps_dynamic_trace, layout$deps_dynamic) +cdl_assert_trace.dynamic <- function(dynamic, spec) { + bad <- setdiff(spec$deps_dynamic_trace, spec$deps_dynamic) if (!length(bad)) { return() } @@ -346,14 +346,14 @@ cdl_assert_trace.dynamic <- function(dynamic, layout) { "existing grouping variables, e.g. map(x, .trace = x) ", "and not map(x, .trace = y). ", "illegal dynamic trace variables for target ", - layout$target, + spec$target, ":\n", multiline_message(bad), call. = FALSE ) } -cdl_assert_trace.default <- function(dynamic, layout) { +cdl_assert_trace.default <- function(dynamic, spec) { character(0) } diff --git a/R/deps.R b/R/deps.R index 3befeb3a1..9be1eb2b7 100644 --- a/R/deps.R +++ b/R/deps.R @@ -78,7 +78,7 @@ deps_target <- function( if (!character_only) { target <- as.character(substitute(target)) } - out <- config$layout[[target]]$deps_build + out <- config$spec[[target]]$deps_build out <- decode_deps_list(out) display_deps_list(select_nonempty(out)) } @@ -217,10 +217,10 @@ deps_profile <- function( old_values <- unname(old_values) elt <- paste(old_values[1], collapse = "") old_values[1] <- config$cache$digest(elt, serialize = FALSE) - layout <- config$layout[[target]] + spec <- config$spec[[target]] new_values <- c( config$cache$digest( - paste(layout$command_standardized, collapse = ""), + paste(spec$command_standardized, collapse = ""), serialize = FALSE ), dependency_hash(target, config), @@ -238,7 +238,7 @@ deps_profile <- function( #' @title List the targets and imports that are reproducibly tracked. #' \lifecycle{stable} -#' @description List all the layout +#' @description List all the spec #' in your project's dependency network. #' @export #' @return A character vector with the names of reproducibly-tracked targets. @@ -258,7 +258,7 @@ tracked <- function(config) { out <- lightly_parallelize( X = V(config$graph)$name, FUN = function(target) { - out <- config$layout[[target]]$deps_build + out <- config$spec[[target]]$deps_build out <- as.list(out) out <- unlist(out) c(out, target) diff --git a/R/drake_build.R b/R/drake_build.R index c26e63985..a4ca1fabd 100644 --- a/R/drake_build.R +++ b/R/drake_build.R @@ -154,8 +154,8 @@ drake_debug <- function( assign(dep, value, envir = config$envir_targets, inherits = FALSE) } } - config$layout[[target]]$command_build <- cdl_preprocess_command( - debug_command(config$layout[[target]]$command) + config$spec[[target]]$command_build <- cdl_preprocess_command( + debug_command(config$spec[[target]]$command) ) config$ht_dynamic <- ht_new() meta <- drake_meta_(target = target, config = config) diff --git a/R/drake_config.R b/R/drake_config.R index 9125fd3a8..06b9e0838 100644 --- a/R/drake_config.R +++ b/R/drake_config.R @@ -146,7 +146,7 @@ #' downgrade `drake` to a compatible version #' rather than rerun all your targets from scratch. #' -#' @param graph An `igraph` object from the previous `make()`. +#' @param graph An `config$graph` object from an existing `drake_config()`. #' Supplying a pre-built graph could save time. #' #' @param trigger Name of the trigger to apply to all targets. @@ -378,10 +378,11 @@ #' target being built, and `config` is a configuration list of #' runtime parameters generated by [drake_config()]. #' -#' @param layout `config$layout`, where `config` is the return value -#' from a prior call to `drake_config()`. If your plan or environment -#' have changed since the last `make()`, do not supply a `layout` argument. -#' Otherwise, supplying one could save time. +#' @param spec `config$spec` from an existing `drake_config()`. +#' Supplying a pre-built workflow specification to this argument +#' could save time in some cases. +#' +#' @param layout Deprecated. Use `spec` instead. #' #' @param lock_envir Logical, whether to lock `config$envir` during `make()`. #' If `TRUE`, `make()` quits in error whenever a command in your @@ -532,6 +533,7 @@ drake_config <- function( sleep = function(i) 0.01, hasty_build = NULL, memory_strategy = "speed", + spec = NULL, layout = NULL, lock_envir = TRUE, history = TRUE, @@ -549,7 +551,7 @@ drake_config <- function( deprecate_arg(pruning_strategy, "pruning_strategy", "memory_strategy") deprecate_arg(timeout, "timeout", "elapsed and/or cpu") deprecate_arg(graph, "graph") - deprecate_arg(layout, "layout") + deprecate_arg(spec, "spec") deprecate_arg(hasty_build, "hasty_build") deprecate_arg(session, "session") deprecate_arg(ensure_workers, "ensure_workers") @@ -558,6 +560,8 @@ drake_config <- function( deprecate_arg(recipe_command, "recipe_command") deprecate_arg(prepend, "prepend") deprecate_arg(makefile_path, "makefile_path") + deprecate_arg(layout, "layout", "spec") # 2019-12-15 + spec <- layout %|||% spec memory_strategy <- match.arg(memory_strategy, choices = memory_strategies()) if (memory_strategy == "memory") { memory_strategy <- "preclean" @@ -584,7 +588,7 @@ drake_config <- function( if (identical(force, TRUE)) { drake_set_session_info(cache = cache, full = session_info) } - layout <- create_drake_layout( + spec <- create_drake_spec( plan = plan, envir = envir, logger = logger, @@ -592,13 +596,13 @@ drake_config <- function( trigger = trigger, cache = cache ) - ht_is_dynamic <- new_ht_is_dynamic(layout) - set_deps_dynamic_whole(layout, ht_is_dynamic) - ht_dynamic_deps <- new_ht_dynamic_deps(layout) + ht_is_dynamic <- new_ht_is_dynamic(spec) + set_deps_dynamic_whole(spec, ht_is_dynamic) + ht_dynamic_deps <- new_ht_dynamic_deps(spec) ht_is_subtarget <- ht_new() # Gets replaced in make(). graph <- create_drake_graph( plan = plan, - layout = layout, + spec = spec, targets = targets, cache = cache, jobs = jobs_preprocess, @@ -628,7 +632,7 @@ drake_config <- function( packages = packages, lib_loc = lib_loc, prework = prework, - layout = layout, + spec = spec, graph = graph, seed = seed, trigger = trigger, @@ -666,49 +670,49 @@ drake_config <- function( out } -new_ht_dynamic_deps <- function(layout) { +new_ht_dynamic_deps <- function(spec) { ht <- ht_new() - lapply(layout, log_ht_dynamic_deps, ht = ht) + lapply(spec, log_ht_dynamic_deps, ht = ht) ht } -log_ht_dynamic_deps <- function(layout, ht) { - ht_set(ht, layout$deps_dynamic) +log_ht_dynamic_deps <- function(spec, ht) { + ht_set(ht, spec$deps_dynamic) } -new_ht_is_dynamic <- function(layout) { +new_ht_is_dynamic <- function(spec) { ht <- ht_new() - lapply(layout, log_ht_is_dynamic, ht = ht) + lapply(spec, log_ht_is_dynamic, ht = ht) ht } -log_ht_is_dynamic <- function(layout, ht) { - if (inherits(layout$dynamic, "dynamic")) { - ht_set(ht, layout$target) +log_ht_is_dynamic <- function(spec, ht) { + if (inherits(spec$dynamic, "dynamic")) { + ht_set(ht, spec$target) } } -set_deps_dynamic_whole <- function(layout, ht_is_dynamic) { +set_deps_dynamic_whole <- function(spec, ht_is_dynamic) { if (!length(ht_list(ht_is_dynamic))) { return() } lapply( - names(layout), + names(spec), set_deps_dynamic_whole1, - layout = layout, + spec = spec, ht_is_dynamic = ht_is_dynamic ) } -set_deps_dynamic_whole1 <- function(target, layout, ht_is_dynamic) { - this_layout <- layout[[target]] - if (this_layout$imported) { +set_deps_dynamic_whole1 <- function(target, spec, ht_is_dynamic) { + this_spec <- spec[[target]] + if (this_spec$imported) { return() } - deps <- this_layout$deps_build$memory + deps <- this_spec$deps_build$memory index <- vapply(deps, ht_exists, ht = ht_is_dynamic, FUN.VALUE = logical(1)) - whole <- setdiff(deps[index], this_layout$deps_dynamic) - layout[[target]]$deps_dynamic_whole <- whole + whole <- setdiff(deps[index], this_spec$deps_dynamic) + spec[[target]]$deps_dynamic_whole <- whole } sanitize_targets <- function(targets, plan) { diff --git a/R/drake_graph_info.R b/R/drake_graph_info.R index e0aa117f1..c09b00b56 100644 --- a/R/drake_graph_info.R +++ b/R/drake_graph_info.R @@ -154,7 +154,7 @@ drake_graph_info <- function( config$hover <- hover config$on_select_col <- on_select_col config$file_out <- lapply(all_targets(config), function(target) { - config$layout[[target]]$deps_build$file_out + config$spec[[target]]$deps_build$file_out }) names(config$file_out) <- all_targets(config) if (!show_output_files) { @@ -311,7 +311,7 @@ get_cluster_grouping <- function(config, group) { vapply( X = config$nodes$id, FUN = function(x) { - out <- config$layout[[x]][[group]] + out <- config$spec[[x]][[group]] if (!is.character(out)) { out <- safe_deparse(out, backtick = TRUE) } @@ -634,7 +634,7 @@ target_hover_text <- function(targets, config) { commands <- vapply( X = targets, FUN = function(target) { - config$layout[[target]]$command_standardized + config$spec[[target]]$command_standardized }, FUN.VALUE = character(1), USE.NAMES = FALSE diff --git a/R/drake_meta_.R b/R/drake_meta_.R index 6663925ab..d26087b8a 100644 --- a/R/drake_meta_.R +++ b/R/drake_meta_.R @@ -24,15 +24,15 @@ drake_meta_impl.subtarget <- function(target, config) { } drake_meta_impl.default <- function(target, config) { - layout <- config$layout[[target]] + spec <- config$spec[[target]] meta <- list( name = target, target = target, - imported = layout$imported %|||% TRUE, + imported = spec$imported %|||% TRUE, missing = target_missing(target, config), - file_out = layout$deps_build$file_out, - format = layout$format %||NA% "none", - dynamic = is.call(layout$dynamic) + file_out = spec$deps_build$file_out, + format = spec$format %||NA% "none", + dynamic = is.call(spec$dynamic) ) if (config$log_build_times) { meta$time_start <- proc_time() @@ -42,7 +42,7 @@ drake_meta_impl.default <- function(target, config) { meta$trigger <- trigger(condition = TRUE) } else { meta$isfile <- FALSE - meta$trigger <- as.list(layout$trigger) + meta$trigger <- as.list(spec$trigger) meta$seed <- resolve_target_seed(target, config) } # For imported files. @@ -52,7 +52,7 @@ drake_meta_impl.default <- function(target, config) { meta$size <- storage_size(path) } if (meta$trigger$command) { - meta$command <- layout$command_standardized + meta$command <- spec$command_standardized } if (meta$trigger$depend) { meta$dependency_hash <- dependency_hash(target = target, config = config) @@ -62,7 +62,7 @@ drake_meta_impl.default <- function(target, config) { meta$output_file_hash <- output_file_hash(target = target, config = config) } if (!is.null(meta$trigger$change)) { - try_load_deps(layout$deps_change$memory, config = config) + try_load_deps(spec$deps_change$memory, config = config) meta$trigger$value <- eval(meta$trigger$change, config$envir_targets) } if (is_dynamic(target, config)) { @@ -100,7 +100,7 @@ target_exists_fast <- Vectorize( ) resolve_target_seed <- function(target, config) { - seed <- config$layout[[target]]$seed + seed <- config$spec[[target]]$seed if (is.null(seed) || is.na(seed)) { seed <- seed_from_basic_types(config$seed, target) } @@ -123,13 +123,13 @@ integer_hash <- function(x, mod = .Machine$integer.max) { } dependency_hash <- function(target, config) { - layout <- config$layout[[target]] - x <- layout$deps_build + spec <- config$spec[[target]] + x <- spec$deps_build deps <- c(x$globals, x$namespaced, x$loadd, x$readd) if (is_imported(target, config)) { deps <- c(deps, x$file_in, x$knitr_in) } - deps <- setdiff(deps, layout$deps_dynamic) + deps <- setdiff(deps, spec$deps_dynamic) if (!length(deps)) { return("") } @@ -141,9 +141,9 @@ dependency_hash <- function(target, config) { } dynamic_dependency_hash <- function(target, config) { - layout <- config$layout[[target]] - deps_dynamic <- layout$deps_dynamic - deps_trace <- sort(unique(layout$deps_dynamic_trace)) + spec <- config$spec[[target]] + deps_dynamic <- spec$deps_dynamic + deps_trace <- sort(unique(spec$deps_dynamic_trace)) deps <- c(deps_dynamic, deps_trace) dependency_hash_impl(deps, config) } @@ -167,7 +167,7 @@ self_hash <- function(target, config) { } is_imported <- function(target, config) { - config$layout[[target]]$imported %|||% TRUE + config$spec[[target]]$imported %|||% TRUE } input_file_hash <- function( @@ -175,7 +175,7 @@ input_file_hash <- function( config, size_threshold = rehash_storage_size_threshold ) { - deps <- config$layout[[target]]$deps_build + deps <- config$spec[[target]]$deps_build files <- sort(unique(as.character(c(deps$file_in, deps$knitr_in)))) if (!length(files)) { return("") @@ -195,7 +195,7 @@ output_file_hash <- function( config, size_threshold = rehash_storage_size_threshold ) { - deps <- config$layout[[target]]$deps_build + deps <- config$spec[[target]]$deps_build files <- sort(unique(as.character(deps$file_out))) if (!length(files)) { return("") diff --git a/R/dynamic.R b/R/dynamic.R index bb4beefe6..15a4992af 100644 --- a/R/dynamic.R +++ b/R/dynamic.R @@ -116,7 +116,7 @@ read_trace <- function( } dynamic_build <- function(target, meta, config) { - subtargets <- config$layout[[target]]$subtargets + subtargets <- config$spec[[target]]$subtargets if (config$log_build_times) { meta$time_command <- proc_time() - meta$time_start } @@ -127,12 +127,12 @@ dynamic_build <- function(target, meta, config) { } append_trace <- function(target, value, config) { - layout <- config$layout[[target]] - if (!length(layout$deps_dynamic_trace)) { + spec <- config$spec[[target]] + if (!length(spec$deps_dynamic_trace)) { return(value) } - dynamic <- layout$dynamic - trace <- get_trace_impl(dynamic, value, layout, config) + dynamic <- spec$dynamic + trace <- get_trace_impl(dynamic, value, spec, config) trace <- subset_trace(trace, config) attr(value, "dynamic_trace") <- trace value @@ -147,47 +147,47 @@ subset_trace <- function(trace, config) { }) } -get_trace_impl <- function(dynamic, value, layout, config) { +get_trace_impl <- function(dynamic, value, spec, config) { UseMethod("get_trace_impl") } -get_trace_impl.map <- function(dynamic, value, layout, config) { +get_trace_impl.map <- function(dynamic, value, spec, config) { trace <- lapply( - layout$deps_dynamic_trace, + spec$deps_dynamic_trace, get, envir = config$envir_targets, inherits = FALSE ) trace <- lapply(trace, chr_dynamic) - names(trace) <- layout$deps_dynamic_trace + names(trace) <- spec$deps_dynamic_trace trace } -get_trace_impl.cross <- function(dynamic, value, layout, config) { +get_trace_impl.cross <- function(dynamic, value, spec, config) { size <- lapply( - layout$deps_dynamic, + spec$deps_dynamic, get_dynamic_size, config = config ) index <- lapply(size, seq_len) - names(index) <- layout$deps_dynamic + names(index) <- spec$deps_dynamic index <- rev(expand.grid(rev(index))) trace <- lapply( - layout$deps_dynamic_trace, + spec$deps_dynamic_trace, get, envir = config$envir_targets, inherits = FALSE ) trace <- lapply(trace, chr_dynamic) - names(trace) <- layout$deps_dynamic_trace - trace <- lapply(layout$deps_dynamic_trace, function(key) { + names(trace) <- spec$deps_dynamic_trace + trace <- lapply(spec$deps_dynamic_trace, function(key) { trace[[key]][index[[key]]] }) - names(trace) <- layout$deps_dynamic_trace + names(trace) <- spec$deps_dynamic_trace trace } -get_trace_impl.group <- function(dynamic, value, layout, config) { +get_trace_impl.group <- function(dynamic, value, spec, config) { by_key <- which_by(dynamic) by_value <- get(by_key, envir = config$envir_targets, inherits = FALSE) trace <- list(unique(by_value)) @@ -220,7 +220,7 @@ register_subtargets <- function(target, static_ok, dynamic_ok, config) { } if (length(subtargets_all)) { register_in_graph(target, subtargets_all, config) - register_in_layout(target, subtargets_all, config) + register_in_spec(target, subtargets_all, config) } ndeps <- length(subtargets_build) if (ndeps) { @@ -274,42 +274,42 @@ register_in_graph <- function(target, subtargets, config) { ) } -register_in_layout <- function(target, subtargets, config) { - layout <- config$layout[[target]] - dynamic <- layout$dynamic +register_in_spec <- function(target, subtargets, config) { + spec <- config$spec[[target]] + dynamic <- spec$dynamic lapply( seq_along(subtargets), - register_subtarget_layout, + register_subtarget_spec, parent = target, subtargets = subtargets, - layout = layout, + spec = spec, dynamic = dynamic, config = config ) - config$layout[[target]]$subtargets <- subtargets + config$spec[[target]]$subtargets <- subtargets } -register_subtarget_layout <- function( +register_subtarget_spec <- function( index, parent, subtargets, dynamic, - layout, + spec, config ) { subtarget <- subtargets[index] - layout$target <- subtarget - layout$subtarget_index <- index - layout$subtarget_parent <- parent - mem_deps <- all.vars(layout$dynamic) - layout$dynamic <- NULL - layout$deps_build$memory <- unique(c(layout$deps_build$memory, mem_deps)) - layout$seed <- seed_from_basic_types(config$seed, layout$seed, subtarget) - layout <- register_dynamic_subdeps(dynamic, layout, index, parent, config) + spec$target <- subtarget + spec$subtarget_index <- index + spec$subtarget_parent <- parent + mem_deps <- all.vars(spec$dynamic) + spec$dynamic <- NULL + spec$deps_build$memory <- unique(c(spec$deps_build$memory, mem_deps)) + spec$seed <- seed_from_basic_types(config$seed, spec$seed, subtarget) + spec <- register_dynamic_subdeps(dynamic, spec, index, parent, config) assign( x = subtarget, - value = layout, - envir = config$layout, + value = spec, + envir = config$spec, inherits = FALSE ) } @@ -342,16 +342,16 @@ dynamic_pad_revdep_keys <- function(target, config) { increase_revdep_keys(config$queue, target, config) } -register_dynamic_subdeps <- function(dynamic, layout, index, parent, config) { +register_dynamic_subdeps <- function(dynamic, spec, index, parent, config) { index_deps <- subtarget_deps(dynamic, parent, index, config) - for (dep in layout$deps_dynamic) { + for (dep in spec$deps_dynamic) { if (is_dynamic(dep, config)) { - subdeps <- config$layout[[dep]]$subtargets[index_deps[[dep]]] - layout$deps_build$memory <- c(layout$deps_build$memory, subdeps) - layout$deps_build$memory <- setdiff(layout$deps_build$memory, dep) + subdeps <- config$spec[[dep]]$subtargets[index_deps[[dep]]] + spec$deps_build$memory <- c(spec$deps_build$memory, subdeps) + spec$deps_build$memory <- setdiff(spec$deps_build$memory, dep) } } - layout + spec } is_dynamic <- function(target, config) { @@ -419,7 +419,7 @@ def_group <- function(..., .by = NULL, .trace = NULL) { # nocov end subtarget_names <- function(target, config) { - dynamic <- config$layout[[target]]$dynamic + dynamic <- config$spec[[target]]$dynamic hashes <- dynamic_hash_list(dynamic, target, config) hashes <- subtarget_hashes(dynamic, target, hashes, config) hashes <- vapply(hashes, shorten_dynamic_hash, FUN.VALUE = character(1)) @@ -445,14 +445,14 @@ dynamic_hash_list <- function(dynamic, target, config) { } dynamic_hash_list.map <- function(dynamic, target, config) { - deps <- sort(config$layout[[target]]$deps_dynamic) + deps <- sort(config$spec[[target]]$deps_dynamic) hashes <- lapply(deps, read_dynamic_hashes, config = config) assert_equal_branches(target, deps, hashes) hashes } dynamic_hash_list.cross <- function(dynamic, target, config) { - deps <- sort(config$layout[[target]]$deps_dynamic) + deps <- sort(config$spec[[target]]$deps_dynamic) lapply(deps, read_dynamic_hashes, config = config) } diff --git a/R/handle_triggers.R b/R/handle_triggers.R index 3f1154a66..4ffe00ec6 100644 --- a/R/handle_triggers.R +++ b/R/handle_triggers.R @@ -279,7 +279,7 @@ trigger_file <- function(target, meta, meta_old, config) { } trigger_file_missing <- function(target, meta, config) { - file_out <- config$layout[[target]]$deps_build$file_out + file_out <- config$spec[[target]]$deps_build$file_out for (file in file_out) { if (!file.exists(config$cache$decode_path(file))) { return(TRUE) @@ -318,7 +318,7 @@ trigger_condition <- function(target, meta, config) { } if (is.language(meta$trigger$condition)) { try_load_deps( - config$layout[[target]]$deps_condition$memory, + config$spec[[target]]$deps_condition$memory, config = config ) value <- eval(meta$trigger$condition, envir = config$envir_targets) diff --git a/R/hpc.R b/R/hpc.R index 30227c605..8ffabb3ea 100644 --- a/R/hpc.R +++ b/R/hpc.R @@ -83,12 +83,12 @@ drake_hpc_template_files <- function() { } no_hpc <- function(target, config) { - identical(config$layout[[target]]$hpc, FALSE) || + identical(config$spec[[target]]$hpc, FALSE) || is_dynamic(target, config) } hpc_caching <- function(target, config) { - out <- config$layout[[target]]$caching + out <- config$spec[[target]]$caching if (is.null(out) || is.na(out)) { out <- config$caching } @@ -98,7 +98,7 @@ hpc_caching <- function(target, config) { hpc_config <- function(config) { discard <- c( "imports", - "layout", + "spec", "plan", "targets", "trigger" @@ -110,30 +110,30 @@ hpc_config <- function(config) { config } -hpc_layout <- function(target, config) { +hpc_spec <- function(target, config) { class(target) <- ifelse(is_subtarget(target, config), "subtarget", "target") - hpc_layout_impl(target, config) + hpc_spec_impl(target, config) } -hpc_layout_impl <- function(target, config) { - UseMethod("hpc_layout_impl") +hpc_spec_impl <- function(target, config) { + UseMethod("hpc_spec_impl") } -hpc_layout_impl.subtarget <- function(target, config) { - layout <- new.env(parent = emptyenv()) - parent <- config$layout[[target]]$subtarget_parent - dynamic_deps <- config$layout[[target]]$deps_dynamic +hpc_spec_impl.subtarget <- function(target, config) { + spec <- new.env(parent = emptyenv()) + parent <- config$spec[[target]]$subtarget_parent + dynamic_deps <- config$spec[[target]]$deps_dynamic keys <- c(target, parent, dynamic_deps) for (key in keys) { - assign(key, config$layout[[key]], envir = layout, inherits = FALSE) + assign(key, config$spec[[key]], envir = spec, inherits = FALSE) } - layout + spec } -hpc_layout_impl.default <- function(target, config) { - layout <- new.env(parent = emptyenv()) - assign(target, config$layout[[target]], envir = layout, inherits = FALSE) - layout +hpc_spec_impl.default <- function(target, config) { + spec <- new.env(parent = emptyenv()) + assign(target, config$spec[[target]], envir = spec, inherits = FALSE) + spec } wait_outfile_checksum <- function(target, checksum, config, timeout = 300) { @@ -196,7 +196,7 @@ is_good_checksum <- function(target, checksum, config) { FUN.VALUE = logical(1) ) ) - format <- config$layout[[target]]$format + format <- config$spec[[target]]$format if (!is.null(format) && !is.na(format)) { format_file <- config$cache$file_return_key(target) out <- out && file.exists(format_file) @@ -228,7 +228,7 @@ get_checksum <- function(target, config) { } get_outfile_checksum <- function(target, config) { - deps <- config$layout[[target]]$deps_build + deps <- config$spec[[target]]$deps_build files <- sort(unique(as.character(deps$file_out))) out <- vapply( X = files, @@ -316,7 +316,7 @@ this_os <- function() { } classify_build <- function(build, config) { - class <- paste0("drake_build_", config$layout[[build$target]]$format) + class <- paste0("drake_build_", config$spec[[build$target]]$format) class(build) <- class build } diff --git a/R/local_build.R b/R/local_build.R index 1b4ab8f97..9055f10ed 100644 --- a/R/local_build.R +++ b/R/local_build.R @@ -55,8 +55,8 @@ try_build <- function(target, meta, config) { return(dynamic_build(target, meta, config)) } retries <- 0L - layout <- config$layout[[target]] %|||% list() - max_retries <- layout$retries + spec <- config$spec[[target]] %|||% list() + max_retries <- spec$retries if (is.null(max_retries) || is.na(max_retries)) { max_retries <- config$retries } @@ -103,11 +103,11 @@ with_seed_timeout <- function(target, meta, config) { } resolve_timeouts <- function(target, config) { - layout <- config$layout[[target]] %|||% list() + spec <- config$spec[[target]] %|||% list() vapply( X = c("cpu", "elapsed"), FUN = function(key) { - out <- layout[[key]] + out <- spec[[key]] if (is.null(out) || is.na(out)) { out <- config[[key]] } @@ -244,7 +244,7 @@ with_call_stack <- function(target, config) { e$calls <- head(sys.calls()[-seq_len(frame + 7)], -2) signalCondition(e) } - expr <- config$layout[[target]]$command_build + expr <- config$spec[[target]]$command_build # Need to make sure the environment is locked for running commands. # Why not just do this once at the beginning of `make()`? # Because do_prework() and future::value() @@ -331,7 +331,7 @@ conclude_build <- function(build, config) { value <- assign_format( target = target, value = value, - format = config$layout[[target]]$format, + format = config$spec[[target]]$format, config = config ) store_outputs(target = target, value = value, meta = meta, config = config) @@ -418,7 +418,7 @@ assign_to_envir <- function(target, value, config) { if (is_subtarget(target, config)) { return() } - memory_strategy <- config$layout[[target]]$memory_strategy + memory_strategy <- config$spec[[target]]$memory_strategy if (is.null(memory_strategy) || is.na(memory_strategy)) { memory_strategy <- config$memory_strategy } @@ -457,7 +457,7 @@ value_format.default <- function(value, target, config) { } assert_output_files <- function(target, meta, config) { - deps <- config$layout[[target]]$deps_build + deps <- config$spec[[target]]$deps_build if (!length(deps$file_out)) { return() } @@ -545,7 +545,7 @@ store_failure <- function(target, meta, config) { set_progress <- function(target, value, config) { skip_progress <- !identical(config$running_make, TRUE) || !config$log_progress || - (config$layout[[target]]$imported %||% FALSE) + (config$spec[[target]]$imported %||% FALSE) if (skip_progress) { return() } diff --git a/R/make.R b/R/make.R index 6cda4bb53..eba678ffa 100644 --- a/R/make.R +++ b/R/make.R @@ -170,6 +170,7 @@ make <- function( hasty_build = NULL, memory_strategy = "speed", layout = NULL, + spec = NULL, lock_envir = TRUE, history = TRUE, recover = FALSE, @@ -227,6 +228,7 @@ make <- function( hasty_build = hasty_build, memory_strategy = memory_strategy, layout = layout, + spec = spec, lock_envir = lock_envir, history = history, recover = recover, diff --git a/R/manage_memory.R b/R/manage_memory.R index 9d36d1807..5481af4f8 100644 --- a/R/manage_memory.R +++ b/R/manage_memory.R @@ -12,7 +12,7 @@ #' @param jobs Number of jobs for local parallel computing manage_memory <- function(target, config, downstream = NULL, jobs = 1) { stopifnot(length(target) == 1L) - memory_strategy <- config$layout[[target]]$memory_strategy + memory_strategy <- config$spec[[target]]$memory_strategy if (is.null(memory_strategy) || is.na(memory_strategy)) { memory_strategy <- config$memory_strategy } @@ -133,7 +133,7 @@ deps_memory <- function(targets, config) { out <- lapply( X = targets, FUN = function(target) { - config$layout[[target]]$deps_build$memory + config$spec[[target]]$deps_build$memory } ) as.character(unlist(out)) @@ -239,10 +239,10 @@ load_subtarget_subdeps <- function(subtarget, config) { if (!is_subtarget(subtarget, config)) { return() } - layout <- config$layout[[subtarget]] - parent <- layout$subtarget_parent - index <- layout$subtarget_index - dynamic <- config$layout[[parent]]$dynamic + spec <- config$spec[[subtarget]] + parent <- spec$subtarget_parent + index <- spec$subtarget_index + dynamic <- config$spec[[parent]]$dynamic deps <- subtarget_deps(dynamic, parent, index, config) load_by_as_subdep(parent, index, config) dep_names <- names(deps) @@ -259,7 +259,7 @@ load_subtarget_subdeps <- function(subtarget, config) { } load_by_as_subdep <- function(parent, index, config) { - dynamic <- config$layout[[parent]]$dynamic + dynamic <- config$spec[[parent]]$dynamic if (no_by(dynamic)) { return() } @@ -287,8 +287,8 @@ load_subtarget_subdep <- function(subtarget, dep, deps, config) { } load_dynamic_subdep <- function(subtarget, dep, index, config) { - parent <- config$layout[[subtarget]]$subtarget_parent - dynamic <- config$layout[[parent]]$dynamic + parent <- config$spec[[subtarget]]$subtarget_parent + dynamic <- config$spec[[parent]]$dynamic load_dynamic_subdep_impl(dynamic, parent, dep, index, config) } @@ -343,7 +343,7 @@ load_static_subdep <- function(dep, index, config) { } sync_envir_dynamic <- function(target, config) { - dynamic_whole <- config$layout[[target]]$deps_dynamic_whole + dynamic_whole <- config$spec[[target]]$deps_dynamic_whole index <- !vlapply( dynamic_whole, exists, diff --git a/R/store_outputs.R b/R/store_outputs.R index 1d1546502..b650cca0d 100644 --- a/R/store_outputs.R +++ b/R/store_outputs.R @@ -30,7 +30,7 @@ store_triggers <- function(target, meta, config) { use_cache = FALSE ) } - store_output_files(config$layout[[target]]$deps_build$file_out, meta, config) + store_output_files(config$spec[[target]]$deps_build$file_out, meta, config) } store_output_files <- function(files, meta, config) { @@ -179,7 +179,7 @@ finalize_meta <- function(target, value, meta, hash, config) { meta$hash <- hash meta$size <- NROW(value) if (is_dynamic(target, config)) { - meta$subtargets <- config$layout[[target]]$subtargets + meta$subtargets <- config$spec[[target]]$subtargets } if (is_dynamic_dep(target, config)) { meta$dynamic_hashes <- dynamic_hashes(value, meta$size, config) @@ -201,9 +201,9 @@ finalize_triggers <- function(target, meta, config) { if (is_subtarget(target, config)) { return(meta) } - layout <- config$layout[[target]] + spec <- config$spec[[target]] if (is.null(meta$command)) { - meta$command <- layout$command_standardized + meta$command <- spec$command_standardized } if (is.null(meta$dependency_hash)) { meta$dependency_hash <- dependency_hash(target = target, config = config) diff --git a/man/cached_planned.Rd b/man/cached_planned.Rd index 971b894f6..3c158ef1b 100644 --- a/man/cached_planned.Rd +++ b/man/cached_planned.Rd @@ -34,6 +34,23 @@ A character vector of target and sub-target names. Includes dynamic sub-targets as well. See examples for details. } +\examples{ +\dontrun{ +isolate_example("cache_planned() example", { +plan <- drake_plan(w = 1) +make(plan) +cached_planned(plan) +plan <- drake_plan( + x = seq_len(2), + y = target(x, dynamic = map(x)) +) +cached_planned(plan) +make(plan) +cached_planned(plan) +cached() +}) +} +} \seealso{ \code{\link[=cached]{cached()}}, \link{cached_unplanned} } diff --git a/man/cached_unplanned.Rd b/man/cached_unplanned.Rd index 9ecc3e820..ff5c68972 100644 --- a/man/cached_unplanned.Rd +++ b/man/cached_unplanned.Rd @@ -34,6 +34,26 @@ A character vector of target and sub-target names. Includes dynamic sub-targets as well. See examples for details. } +\examples{ +\dontrun{ +isolate_example("cache_unplanned() example", { +plan <- drake_plan(w = 1) +make(plan) +cached_unplanned(plan) +plan <- drake_plan( + x = seq_len(2), + y = target(x, dynamic = map(x)) +) +cached_unplanned(plan) +make(plan) +cached_unplanned(plan) +# cached_unplanned() helps clean superfluous targets. +cached() +clean(list = cached_unplanned(plan)) +cached() +}) +} +} \seealso{ \code{\link[=cached]{cached()}}, \link{cached_planned} } diff --git a/man/cmq_build.Rd b/man/cmq_build.Rd index 6bfc33bab..0b3c9506a 100644 --- a/man/cmq_build.Rd +++ b/man/cmq_build.Rd @@ -5,7 +5,7 @@ \title{Build a target using the clustermq backend \lifecycle{maturing}} \usage{ -cmq_build(target, meta, deps, layout, ht_is_subtarget, config) +cmq_build(target, meta, deps, spec, ht_is_subtarget, config) } \arguments{ \item{target}{Target name.} @@ -14,7 +14,7 @@ cmq_build(target, meta, deps, layout, ht_is_subtarget, config) \item{deps}{Named list of target dependencies.} -\item{layout}{Internal, part of the full \code{config$layout}.} +\item{spec}{Internal, part of the full \code{config$spec}.} \item{ht_is_subtarget}{Internal, part of \code{config}} diff --git a/man/drake_config.Rd b/man/drake_config.Rd index 047dc2c25..7533e067a 100644 --- a/man/drake_config.Rd +++ b/man/drake_config.Rd @@ -50,6 +50,7 @@ drake_config( sleep = function(i) 0.01, hasty_build = NULL, memory_strategy = "speed", + spec = NULL, layout = NULL, lock_envir = TRUE, history = TRUE, @@ -193,7 +194,7 @@ and slows down \code{make()} a little. If you need to reduce or limit the number of files in the cache, call \code{make(log_progress = FALSE, recover = FALSE)}.} -\item{graph}{An \code{igraph} object from the previous \code{make()}. +\item{graph}{An \code{config$graph} object from an existing \code{drake_config()}. Supplying a pre-built graph could save time.} \item{trigger}{Name of the trigger to apply to all targets. @@ -432,10 +433,11 @@ help file examples of \code{\link[=drake_envir]{drake_envir()}}. Also see the \code{garbage_collection} argument of \code{make()} and \code{drake_config()}.} -\item{layout}{\code{config$layout}, where \code{config} is the return value -from a prior call to \code{drake_config()}. If your plan or environment -have changed since the last \code{make()}, do not supply a \code{layout} argument. -Otherwise, supplying one could save time.} +\item{spec}{\code{config$spec} from an existing \code{drake_config()}. +Supplying a pre-built workflow specification to this argument +could save time in some cases.} + +\item{layout}{Deprecated. Use \code{spec} instead.} \item{lock_envir}{Logical, whether to lock \code{config$envir} during \code{make()}. If \code{TRUE}, \code{make()} quits in error whenever a command in your diff --git a/man/future_build.Rd b/man/future_build.Rd index b92a80162..d95e42512 100644 --- a/man/future_build.Rd +++ b/man/future_build.Rd @@ -5,7 +5,7 @@ \title{Task passed to individual futures in the \code{"future"} backend \lifecycle{stable}} \usage{ -future_build(target, meta, config, layout, ht_is_subtarget, protect) +future_build(target, meta, config, spec, ht_is_subtarget, protect) } \arguments{ \item{target}{Name of the target.} diff --git a/man/make.Rd b/man/make.Rd index eb0f569c5..15b384f31 100644 --- a/man/make.Rd +++ b/man/make.Rd @@ -52,6 +52,7 @@ make( hasty_build = NULL, memory_strategy = "speed", layout = NULL, + spec = NULL, lock_envir = TRUE, history = TRUE, recover = FALSE, @@ -197,7 +198,7 @@ give you an opportunity to downgrade \code{drake} to a compatible version rather than rerun all your targets from scratch.} -\item{graph}{An \code{igraph} object from the previous \code{make()}. +\item{graph}{An \code{config$graph} object from an existing \code{drake_config()}. Supplying a pre-built graph could save time.} \item{trigger}{Name of the trigger to apply to all targets. @@ -450,10 +451,11 @@ help file examples of \code{\link[=drake_envir]{drake_envir()}}. Also see the \code{garbage_collection} argument of \code{make()} and \code{drake_config()}.} -\item{layout}{\code{config$layout}, where \code{config} is the return value -from a prior call to \code{drake_config()}. If your plan or environment -have changed since the last \code{make()}, do not supply a \code{layout} argument. -Otherwise, supplying one could save time.} +\item{layout}{Deprecated. Use \code{spec} instead.} + +\item{spec}{\code{config$spec} from an existing \code{drake_config()}. +Supplying a pre-built workflow specification to this argument +could save time in some cases.} \item{lock_envir}{Logical, whether to lock \code{config$envir} during \code{make()}. If \code{TRUE}, \code{make()} quits in error whenever a command in your diff --git a/man/tracked.Rd b/man/tracked.Rd index 17e017139..e180f3040 100644 --- a/man/tracked.Rd +++ b/man/tracked.Rd @@ -14,7 +14,7 @@ tracked(config) A character vector with the names of reproducibly-tracked targets. } \description{ -List all the layout +List all the spec in your project's dependency network. } \examples{ diff --git a/tests/testthat/test-3-commands.R b/tests/testthat/test-3-commands.R index 1f2ffb13c..8e60a0596 100644 --- a/tests/testthat/test-3-commands.R +++ b/tests/testthat/test-3-commands.R @@ -19,17 +19,17 @@ test_with_dir("changes to commands are handled well", { character(0) ) config$plan$command[[2]] <- parse(text = "f(1+ 1) # nothing should rebuild") - config$layout <- create_drake_layout( + config$spec <- create_drake_spec( plan = config$plan, envir = config$envir, cache = config$cache, logger = config$logger - )$layout + )$spec testrun(config) nobuild(config) config$plan$command[[2]] <- parse(text = "f(1+ 1 -2 + 2) -1 + 1 #only yourinput changed") - config$layout <- create_drake_layout( + config$spec <- create_drake_spec( plan = config$plan, envir = config$envir, cache = config$cache, @@ -48,12 +48,12 @@ test_with_dir("changes to commands are handled well", { expect_equal(justbuilt(config), "yourinput") config$plan$command[[2]] <- parse(text = "f(1+2) # now downstream should rebuild") - config$layout <- create_drake_layout( + config$spec <- create_drake_spec( plan = config$plan, envir = config$envir, cache = config$cache, logger = config$logger - )$layout + )$spec testrun(config) expect_equal( justbuilt(config), @@ -70,12 +70,12 @@ test_with_dir("changes to commands are handled well", { # command changed for an intermediate file config$plan$command[[1]] <- parse(text = "saveRDS(combined + 1, file_out(\"intermediatefile.rds\"))") - config$layout <- create_drake_layout( + config$spec <- create_drake_spec( plan = config$plan, envir = config$envir, cache = config$cache, logger = config$logger - )$layout + )$spec testrun(config) expect_equal( justbuilt(config), diff --git a/tests/testthat/test-6-hpc.R b/tests/testthat/test-6-hpc.R index b56babd8d..26d0fecf4 100644 --- a/tests/testthat/test-6-hpc.R +++ b/tests/testthat/test-6-hpc.R @@ -137,7 +137,7 @@ test_with_dir("parallelism can be a scheduler function", { plan <- drake_plan(x = file.create("x")) build_ <- function(target, config) { tidy_expr <- eval( - expr = config$layout[[target]]$command_build, + expr = config$spec[[target]]$command_build, envir = config$envir_targets ) eval(expr = tidy_expr, envir = config$envir_targets) diff --git a/tests/testthat/test-6-plans.R b/tests/testthat/test-6-plans.R index b650bb07b..d1a615a29 100644 --- a/tests/testthat/test-6-plans.R +++ b/tests/testthat/test-6-plans.R @@ -194,7 +194,7 @@ test_with_dir( ) }) expect_true(all(letters[1:4] %in% cached())) - expect_true(all(letters[1:4] %in% names(con$layout))) + expect_true(all(letters[1:4] %in% names(con$spec))) }) test_with_dir("plans can start with bad symbols", { @@ -204,7 +204,7 @@ test_with_dir("plans can start with bad symbols", { command = 1) y <- drake_config(x) out <- sort(c("a.x.", "b.x.", "X_a", "a....")) - expect_true(all(out %in% names(y$layout))) + expect_true(all(out %in% names(y$spec))) }) test_with_dir("can use semicolons for multi-line commands", { diff --git a/tests/testthat/test-6-triggers.R b/tests/testthat/test-6-triggers.R index b2b4e5d50..52174761b 100644 --- a/tests/testthat/test-6-triggers.R +++ b/tests/testthat/test-6-triggers.R @@ -384,8 +384,8 @@ test_with_dir("deps load into memory for complex triggers", { expect_true(all(plan$target %in% cached())) } config <- drake_config(plan) - expect_equal(config$layout[["psi_2"]]$deps_condition$memory, "psi_1") - expect_equal(config$layout[["psi_3"]]$deps_change$memory, "psi_2") + expect_equal(config$spec[["psi_2"]]$deps_condition$memory, "psi_1") + expect_equal(config$spec[["psi_3"]]$deps_change$memory, "psi_2") }) test_with_dir("trigger components react appropriately", { @@ -870,10 +870,10 @@ test_with_dir("files are collected/encoded from all triggers", { ) ) config <- drake_config(plan) - deps_build <- redecode_path(unlist(config$layout[["x"]]$deps_build)) + deps_build <- redecode_path(unlist(config$spec[["x"]]$deps_build)) deps_condition <- redecode_path( - unlist(config$layout[["x"]]$deps_condition)) - deps_change <- redecode_path(unlist(config$layout[["x"]]$deps_change)) + unlist(config$spec[["x"]]$deps_condition)) + deps_change <- redecode_path(unlist(config$spec[["x"]]$deps_change)) expect_equal( sort(deps_build), sort(c("command_in", "command_out", "command_knitr_in")) diff --git a/tests/testthat/test-7-deprecate.R b/tests/testthat/test-7-deprecate.R index 3def60c3a..12aae567e 100644 --- a/tests/testthat/test-7-deprecate.R +++ b/tests/testthat/test-7-deprecate.R @@ -171,7 +171,7 @@ test_with_dir("deprecate misc utilities", { cache, log_progress = TRUE, init_common_values = TRUE )) load_mtcars_example() - expect_warning(config <- drake_config(my_plan, graph = 1, layout = 2)) + expect_warning(config <- drake_config(my_plan, graph = 1, spec = 2)) expect_warning(make_imports(config)) expect_warning(make_targets(config)) expect_warning(make_with_config(config)) diff --git a/tests/testthat/test-8-cache.R b/tests/testthat/test-8-cache.R index 76c139e8d..05068c915 100644 --- a/tests/testthat/test-8-cache.R +++ b/tests/testthat/test-8-cache.R @@ -86,7 +86,7 @@ test_with_dir("dependency profile", { expect_true(as.logical(dp[dp$name == "depend", "changed"])) expect_equal(sum(dp$changed), 1) plan$command <- "b + c" - config$layout <- create_drake_layout( + config$spec <- create_drake_spec( plan = plan, envir = config$envir, cache = config$cache, diff --git a/tests/testthat/test-9-dynamic.R b/tests/testthat/test-9-dynamic.R index e81dd7bfb..e7afd6514 100644 --- a/tests/testthat/test-9-dynamic.R +++ b/tests/testthat/test-9-dynamic.R @@ -11,13 +11,13 @@ test_with_dir("dynamic dependency detection", { z = target(w, dynamic = group(x, y, .by = c(w, nope))) ) config <- drake_config(plan) - layout <- config$layout + spec <- config$spec config$ht_is_subtarget <- ht_new() - expect_equal(layout[["v"]]$deps_dynamic, character(0)) - expect_equal(layout[["w"]]$deps_dynamic, character(0)) - expect_equal(sort(layout[["x"]]$deps_dynamic), sort(c("indices", "v"))) - expect_equal(sort(layout[["y"]]$deps_dynamic), sort(c("v", "x", "y"))) - expect_equal(sort(layout[["z"]]$deps_dynamic), sort(c("w", "x", "y"))) + expect_equal(spec[["v"]]$deps_dynamic, character(0)) + expect_equal(spec[["w"]]$deps_dynamic, character(0)) + expect_equal(sort(spec[["x"]]$deps_dynamic), sort(c("indices", "v"))) + expect_equal(sort(spec[["y"]]$deps_dynamic), sort(c("v", "x", "y"))) + expect_equal(sort(spec[["z"]]$deps_dynamic), sort(c("w", "x", "y"))) meta1 <- drake_meta_("v", config) meta2 <- drake_meta_("x", config) con2 <- drake_config(drake_plan(x = 1)) @@ -62,18 +62,18 @@ test_with_dir("dynamic sub-target indices", { config$ht_dynamic_size <- ht_new() for (i in seq_len(4)) { ew <- list(u = i, v = i) - expect_equal(subtarget_deps(config$layout$w$dynamic, "w", i, config), ew) + expect_equal(subtarget_deps(config$spec$w$dynamic, "w", i, config), ew) } for (i in seq_len(4)) { for (j in seq_len(4)) { ey <- list(u = i, v = j) k <- 4 * (i - 1) + j - expect_equal(subtarget_deps(config$layout$y$dynamic, "y", k, config), ey) + expect_equal(subtarget_deps(config$spec$y$dynamic, "y", k, config), ey) } } for (i in seq_len(4)) { ez <- list(y = seq(from = 4 * (i - 1) + 1, 4 * i)) - expect_equal(subtarget_deps(config$layout$z$dynamic, "z", i, config), ez) + expect_equal(subtarget_deps(config$spec$z$dynamic, "z", i, config), ez) } }) @@ -1555,9 +1555,9 @@ test_with_dir("whole dynamic targets (#1107)", { results = list(means, sds) ) config <- drake_config(plan, memory_strategy = memory_strategy) - expect_equal(config$layout[["raw"]]$deps_dynamic_whole, character(0)) - expect_equal(config$layout[["rows"]]$deps_dynamic_whole, character(0)) - expect_equal(config$layout[["means"]]$deps_dynamic_whole, "rows") + expect_equal(config$spec[["raw"]]$deps_dynamic_whole, character(0)) + expect_equal(config$spec[["rows"]]$deps_dynamic_whole, character(0)) + expect_equal(config$spec[["means"]]$deps_dynamic_whole, "rows") make( plan, memory_strategy = memory_strategy,