Skip to content

Commit

Permalink
Test some dynamic file recovery
Browse files Browse the repository at this point in the history
  • Loading branch information
@wlandau-lilly
wlandau-lilly committed Feb 21, 2020
1 parent 07484fb commit c506922
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 33 deletions.
42 changes: 22 additions & 20 deletions R/drake_meta_.R
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
drake_meta_ <- function(target, config) {
class(target) <- drake_meta_class(target, config)
meta <- drake_meta_impl(target, config)
meta_old <- NULL
if (target_exists(target, config)) {
meta_old <- config$cache$get(key = target, namespace = "meta")
}
out <- drake_meta_impl(target, meta_old, config)
class(out) <- c("drake_meta", "drake")
out
meta <- subsume_old_meta(target, meta, meta_old, config)
class(meta) <- c("drake_meta", "drake")
meta
}

#' @export
Expand Down Expand Up @@ -42,16 +43,15 @@ drake_meta_class <- function(target, config) {
"static"
}

drake_meta_impl <- function(target, meta_old, config) {
drake_meta_impl <- function(target, config) {
UseMethod("drake_meta_impl")
}

drake_meta_impl.imported_file <- function(target, meta_old, config) { # nolint
drake_meta_impl.imported_file <- function(target, config) { # nolint
spec <- config$spec[[target]]
meta <- list(
name = target,
target = target,
meta_old = meta_old,
imported = TRUE,
isfile = TRUE,
format = "none",
Expand All @@ -66,12 +66,11 @@ drake_meta_impl.imported_file <- function(target, meta_old, config) { # nolint
meta
}

drake_meta_impl.imported_object <- function(target, meta_old, config) { # nolint
drake_meta_impl.imported_object <- function(target, config) { # nolint
spec <- config$spec[[target]]
meta <- list(
name = target,
target = target,
meta_old = meta_old,
imported = TRUE,
isfile = FALSE,
dynamic = FALSE,
Expand All @@ -84,24 +83,22 @@ drake_meta_impl.imported_object <- function(target, meta_old, config) { # nolint
meta
}

drake_meta_impl.subtarget <- function(target, meta_old, config) {
drake_meta_impl.subtarget <- function(target, config) {
list(
name = target,
target = target,
meta_old = meta_old,
imported = FALSE,
isfile = FALSE,
seed = resolve_target_seed(target, config),
time_start = drake_meta_start(config)
)
}

drake_meta_impl.dynamic <- function(target, meta_old, config) {
drake_meta_impl.dynamic <- function(target, config) {
spec <- config$spec[[target]]
meta <- list(
name = target,
target = target,
meta_old = meta_old,
imported = FALSE,
isfile = FALSE,
dynamic = TRUE,
Expand All @@ -115,12 +112,11 @@ drake_meta_impl.dynamic <- function(target, meta_old, config) {
decorate_trigger_meta(target, meta, spec, config)
}

drake_meta_impl.static <- function(target, meta_old, config) {
drake_meta_impl.static <- function(target, config) {
spec <- config$spec[[target]]
meta <- list(
name = target,
target = target,
meta_old = meta_old,
imported = FALSE,
isfile = FALSE,
dynamic = FALSE,
Expand Down Expand Up @@ -150,9 +146,13 @@ decorate_trigger_meta <- function(target, meta, spec, config) {
try_load_deps(spec$deps_change$memory, config = config)
meta$trigger$value <- eval(meta$trigger$change, config$envir_targets)
}
meta
}

subsume_old_meta <- function(target, meta, meta_old, config) {
meta$meta_old <- meta_old
class(target) <- meta$format
meta <- decorate_trigger_format_meta(target, meta, config)
meta
}

decorate_trigger_format_meta <- function(target, meta, config) {
Expand All @@ -167,16 +167,18 @@ decorate_trigger_format_meta.file <- function(target, meta, config) {
if (is.null(meta$meta_old) || !meta$trigger$file) {
return(meta)
}
path <- meta$meta_old$format_file_path
path <- as.character(meta$meta_old$format_file_path)
new_mtime <- storage_mtime(path)
new_size <- storage_size(path)
hash <- meta$meta_old$format_file_hash
should_rehash <- file.exists(path) & should_rehash_local(
hash <- as.character(meta$meta_old$format_file_hash)
exists <- file.exists(path)
hash[!exists] <- ""
should_rehash <- exists & should_rehash_local(
size_threshold = rehash_storage_size_threshold,
new_mtime = new_mtime,
old_mtime = meta$meta_old$format_file_time,
old_mtime = as.numeric(meta$meta_old$format_file_time),
new_size = new_size,
old_size = meta$meta_old$format_file_size
old_size = as.numeric(meta$meta_old$format_file_size)
)
hash[should_rehash] <- rehash_local(path[should_rehash], config)
meta$format_file_path <- path
Expand Down
7 changes: 5 additions & 2 deletions R/handle_triggers.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,12 @@ recovery_key_impl.default <- function(target, meta, config) {
meta$dependency_hash,
meta$input_file_hash,
meta$output_file_hash,
meta$format_file_path,
meta$format_file_hash,
meta$trigger$mode,
meta$format,
as.character(meta$seed),
safe_deparse(meta$trigger$condition, backtick = TRUE),
meta$trigger$mode,
change_hash
)
x <- paste(x, collapse = "|")
Expand Down Expand Up @@ -288,7 +291,7 @@ check_trigger_dynamic <- function(target, meta, config) {

check_trigger_format_file <- function(target, meta, config) {
if (identical(meta$trigger$file, TRUE) && meta$format == "file") {
if (trigger_format_file(target, config)) {
if (trigger_format_file(target, meta, config)) {
config$logger$disk("trigger file (format file)", target = target)
return(TRUE)
}
Expand Down
11 changes: 1 addition & 10 deletions tests/testthat/test-8-decorated-storr.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -877,7 +877,6 @@ test_with_dir("global rds format + target qs (#1124)", {
})

test_with_dir("file format with flat files and static targets (#1168)", {
skip("not ready yet")
skip_on_cran()
write_lines <- function(files, ...) {
for (file in files) {
Expand Down Expand Up @@ -963,7 +962,7 @@ test_with_dir("file format with flat files and static targets (#1168)", {
})

test_with_dir("file format with directories and static targets (#1168)", {
skip("not ready yet")
skip_on_cran()
write_lines <- function(files, ...) {
for (file in files) {
if (!dir.exists(file)) {
Expand Down Expand Up @@ -1058,7 +1057,6 @@ test_with_dir("bad file format value", {
})

test_with_dir("file trigger and dynamic files (#1168)", {
skip("not ready yet")
skip_on_cran()
write_lines <- function(files, ...) {
for (file in files) {
Expand All @@ -1080,7 +1078,6 @@ test_with_dir("file trigger and dynamic files (#1168)", {
})

test_with_dir("data recovery and dynamic files (#1168)", {
skip("not ready yet")
skip_on_cran()
write_lines <- function(files, ...) {
for (file in files) {
Expand All @@ -1098,12 +1095,6 @@ test_with_dir("data recovery and dynamic files (#1168)", {
make(plan)
unlink("no_recover")
config <- drake_config(plan)
# Clean and recover.
clean()
make(plan, recover = TRUE)
expect_equal(justbuilt(config), "x")
expect_false(file.exists("no_recover"))
expect_true(file.exists("b"))
# Clean, remove output file, and fail to recover.
clean()
unlink(c("no_recover", "b"))
Expand Down
1 change: 0 additions & 1 deletion tests/testthat/test-9-dynamic.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1782,7 +1782,6 @@ test_with_dir("target-specific max_expand (#1175)", {
})

test_with_dir("dynamic files + dynamic branching (#1168)", {
skip("not ready yet")
skip_on_cran()
write_lines <- function(files, ...) {
for (file in files) {
Expand Down

1 comment on commit c506922

@lintr-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

R/drake_meta_.R:162:1: style: Variable and function names should not be longer than 30 characters.

​decorate_trigger_format_meta.default <- function(target, meta, config) {
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

R/drake_meta_.R:166:1: style: Variable and function names should not be longer than 30 characters.

​decorate_trigger_format_meta.file <- function(target, meta, config) {
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Please sign in to comment.