Fix Ingest return value

R/util-ApplySpec.R

@@ -35,16 +35,20 @@ ApplySpec <- function(dfSource, columnSpecs, domain) {
   }
 
   # write a query to select the columns from the source
-  columnMapping <- columnSpecs %>% imap(
-    function(spec, name) {
-      mapping <- list(target = name)
-      mapping$source <- spec$source_col %||% name
-      mapping$type <- spec$type %||% NULL
-      return(mapping)
-    }
-  )
+  columnMapping <- columnSpecs %>%
+    imap(
+      function(spec, name) {
+        mapping <- list(target = name)
+        mapping$source <- spec$source_col %||% name
+        mapping$type <- spec$type %||% NULL
+        mapping$required <- spec$required %||% FALSE
+        return(mapping)
+      }
+    ) %>%
+    # Drop non-required columns that aren't in dfSource.
+    purrr::keep(~.x$required || .x$source %in% colnames(dfSource))
 
-  # check that the source columns exists in the source data
+  # check that the required columns exists in the source data
   sourceCols <- columnMapping %>% map("source")
   if (!all(sourceCols %in% names(dfSource))) {
     missingCols <- sourceCols[!sourceCols %in% names(dfSource)]

R/util-CombineSpecs.R

@@ -99,9 +99,9 @@ combine_domain <- function(domain_specs) {
 #' @return A list containing the updated column specification.
 #' @keywords internal
 update_column <- function(existing_col, new_col, col_name) {
-  if (!is.null(existing_col)) {
+  if (length(existing_col)) {
     # Handle required conflict
-    existing_col$required <- existing_col$required || new_col$required
+    existing_col$required <- isTRUE(existing_col$required) || isTRUE(new_col$required)
 
     # Handle type conflict with a warning when available
     if (!is.null(existing_col$type) && !is.null(new_col$type)) {

R/util-Ingest.R

@@ -10,7 +10,7 @@
 #' @param lSpec `list` A named list of column specifications.
 #' @param strDomain `character` Domain name to add to the data frames after ingestions. Default: "Raw"
 #'
-#'  @return `list` A named list of data frames, where each data frame corresponds to a domain in the
+#' @return `list` A named list of data frames, where each data frame corresponds to a domain in the
 #' specification.
 #'
 #' @examples

tests/testthat/test-util-CombineSpecs.R

@@ -202,3 +202,108 @@ test_that("warning if type doesn't match first instance", {
 
   expect_equal(combined, expected)
 })
+
+test_that("CombineSpecs works with optoinal columns", {
+  spec1 <- list(
+    df1 = list(
+      col1 = list(required = TRUE)
+    )
+  )
+  spec2 <- list(
+    df1 = list(
+      col2 = list(required = TRUE)
+    )
+  )
+  expected <- list(
+    df1 = list(
+      col1 = list(required = TRUE),
+      col2 = list(required = TRUE)
+    )
+  )
+  expect_no_error({
+    test_result <- CombineSpecs(list(spec1, spec2), bIsWorkflow = FALSE)
+  })
+  expect_identical(test_result, expected)
+
+  spec1 <- list(
+    df1 = list(
+      col1 = list(required = FALSE)
+    )
+  )
+  spec2 <- list(
+    df1 = list(
+      col2 = list(required = TRUE)
+    )
+  )
+  expected <- list(
+    df1 = list(
+      col1 = list(required = FALSE),
+      col2 = list(required = TRUE)
+    )
+  )
+  expect_no_error({
+    test_result <- CombineSpecs(list(spec1, spec2), bIsWorkflow = FALSE)
+  })
+  expect_identical(test_result, expected)
+
+  spec1 <- list(
+    df1 = list(
+      col1 = list(required = TRUE)
+    )
+  )
+  spec2 <- list(
+    df1 = list(
+      col2 = list(required = FALSE)
+    )
+  )
+  expected <- list(
+    df1 = list(
+      col1 = list(required = TRUE),
+      col2 = list(required = FALSE)
+    )
+  )
+  expect_no_error({
+    test_result <- CombineSpecs(list(spec1, spec2), bIsWorkflow = FALSE)
+  })
+  expect_identical(test_result, expected)
+
+  spec1 <- list(
+    df1 = list(
+      col1 = list(required = FALSE)
+    )
+  )
+  spec2 <- list(
+    df1 = list(
+      col1 = list(required = TRUE)
+    )
+  )
+  expected <- list(
+    df1 = list(
+      col1 = list(required = TRUE)
+    )
+  )
+  expect_no_error({
+    test_result <- CombineSpecs(list(spec1, spec2), bIsWorkflow = FALSE)
+  })
+  expect_identical(test_result, expected)
+
+  spec1 <- list(
+    df1 = list(
+      col1 = list(required = TRUE)
+    )
+  )
+  spec2 <- list(
+    df1 = list(
+      col1 = list(required = FALSE)
+    )
+  )
+  expected <- list(
+    df1 = list(
+      col1 = list(required = TRUE)
+    )
+  )
+  expect_no_error({
+    test_result <- CombineSpecs(list(spec1, spec2), bIsWorkflow = FALSE)
+  })
+  expect_identical(test_result, expected)
+})

tests/testthat/test-util-Ingest.R

@@ -0,0 +1,22 @@
+test_that("Ingest works with optional columns", {
+  lSourceData <- list(
+    df1 = data.frame(
+      a = 1:10,
+      b = letters[1:10]
+    )
+  )
+  lSpec <- list(
+    df1 = list(
+      a = list(required = TRUE, type = "integer"),
+      b = list(required = TRUE, type = "character"),
+      c = list(required = FALSE, type = "character")
+    )
+  )
+  expect_no_error(expect_message(expect_message({
+    test_result <- Ingest(lSourceData, lSpec)
+  })))
+  expected_result <- list(
+    Raw_df1 = lSourceData$df1
+  )
+  expect_identical(test_result, expected_result)
+})