Changing to implicates = 1 in prepare() to avoid potential over-fitti…

…ng issues
ummel · Feb 12, 2024 · a5dc917 · a5dc917
1 parent 02b567e
commit a5dc917
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 13 deletions.
diff --git a/fusion/AHS/2019/2019/input/AHS_2019_2019_input.R b/fusion/AHS/2019/2019/input/AHS_2019_2019_input.R
@@ -2,7 +2,7 @@ library(fusionData)
 library(fusionModel)
 
 # Number of cores to use
-ncores <- 1
+ncores <- 3
 
 # Donor and recipient survey identifiers
 donor <- "AHS_2019"
@@ -19,7 +19,7 @@ dir <- paste("fusion", sub("_", "/", donor), acs.vintage, "input", sep = "/")
 prep <- prepare(donor = donor,
                 recipient = recipient,
                 respondent = "household",
-                implicates = 5)
+                implicates = 1)
 
 # Specify fusion variables to be retained in harmonization results
 # Removed pca for prep 
@@ -73,14 +73,14 @@ saveRDS(prep, file = file.path(dir, paste(donor, acs.vintage, "prep.rds", sep =
 pred.vars <- attr(prep, "xpredictors")
 
 # Set cores for 'fst' to use when writing to disk
-threads_fst(ncores)
+fst::threads_fst(ncores)
 
 # Save training data to disk
 data[[1]] %>%
   select(one_of(c("weight", unlist(prep$y), pred.vars))) %>%
-  write_fst(path = file.path(dir, paste(donor, acs.vintage, "train.fst", sep = "_")), compress = 100)
+  fst::write_fst(path = file.path(dir, paste(donor, acs.vintage, "train.fst", sep = "_")), compress = 100)
 
 # Save prediction data to disk
 data[[2]] %>%
   select(one_of(pred.vars)) %>%
-  write_fst(path = file.path(dir, paste(donor, acs.vintage, "predict.fst", sep = "_")), compress = 100)
+  fst::write_fst(path = file.path(dir, paste(donor, acs.vintage, "predict.fst", sep = "_")), compress = 100)
diff --git a/fusion/CEI/2015-2019/2019/input/CEI_2015-2019_2019_input.R b/fusion/CEI/2015-2019/2019/input/CEI_2015-2019_2019_input.R
@@ -2,7 +2,7 @@ library(fusionData)
 library(fusionModel)
 library(tidyverse)
 library(fst)
-
+library(data.table)
 
 
 # Number of cores to use
@@ -34,7 +34,7 @@ fusion.vars <- googlesheets4::read_sheet("13GRKkVZXapHtP7oK1WUh0Yu7OQ_9icd17wUGu
 prep <- prepare(donor = "CEI_2015-2019",
                 recipient = "ACS_2019",
                 respondent = "household",
-                implicates = 5)
+                implicates = 1)
 
 data <- assemble(prep,
                  fusion.variables = fusion.vars,

diff --git a/fusion/NHTS/2017/2015/input/NHTS_2017_2015_input.R b/fusion/NHTS/2017/2015/input/NHTS_2017_2015_input.R
@@ -19,7 +19,7 @@ dir <- paste("fusion", sub("_", "/", donor), acs.vintage, "input", sep = "/")
 prep <- prepare(donor = donor,
                 recipient = recipient,
                 respondent = "household",
-                implicates = 5)
+                implicates = 1)
 
 # Specify fusion variables to be retained in harmonization results
 data <- assemble(prep,

diff --git a/fusion/NHTS/2017/2017/input/NHTS_2017_2017_input.R b/fusion/NHTS/2017/2017/input/NHTS_2017_2017_input.R
@@ -1,6 +1,7 @@
 library(fusionData)
 library(fusionModel)
-
+library(fst)
+
 # Number of cores to use
 ncores <- 3
 
@@ -19,11 +20,12 @@ dir <- paste("fusion", sub("_", "/", donor), acs.vintage, "input", sep = "/")
 prep <- prepare(donor = donor,
                 recipient = recipient,
                 respondent = "household",
-                implicates = 5)
+                implicates = 1)
 
 # Specify fusion variables to be retained in harmonization results
 data <- assemble(prep,
-                 fusion.variables = c('bike2save','walk2save','price','place','gstotcst','ptrans'),
+                 fusion.variables = c('bike2save','walk2save',
+                                      'price','place','gstotcst','ptrans'),
                  window = 2)
 
 rm(prep)

diff --git a/fusion/RECS/2015/2015/input/RECS_2015_2015_input.R b/fusion/RECS/2015/2015/input/RECS_2015_2015_input.R
@@ -19,7 +19,7 @@ dir <- paste("fusion", sub("_", "/", donor), acs.vintage, "input", sep = "/")
 prep <- prepare(donor = donor,
                 recipient = recipient,
                 respondent = "household",
-                implicates = 5)
+                implicates = 1)
 
 # Specify fusion variables to be retained in harmonization results
 # Removed pca for prep 

diff --git a/fusion/RECS/2015/2015/output/RECS_2015_2015_output.R b/fusion/RECS/2015/2015/output/RECS_2015_2015_output.R
@@ -7,7 +7,7 @@ dir <- "fusion/RECS/2015/2015/input"
 ncores <- 1
 
 # Number of implicates to generate
-nimps <- 30
+nimps <- 40
 
 # Output files path stub
 out.path <- file.path(sub("input", "output", dir), sub("train.fst$", "", list.files(dir, "train\\.fst$")))