Create gridded file (#153)

* need to fix gridded data loading * fixed upload for gridded data table * update input options for alt specific variables for conditional logit models * model design works for single dimension grid variable in conditional logit model * success running conditional logit with 1-D grid data table * added info widget to model name input * changed naming convention for model output by removing the expected catch names from the name, now they match model design names * fixed welfare_predict function for multiple alt var variables
noaa-nwfsc · Aug 6, 2024 · 4da880c · 4da880c
1 parent 66494db
commit 4da880c
Show file tree

Hide file tree

Showing 10 changed files with 140 additions and 105 deletions.
diff --git a/R/create_model_input.R b/R/create_model_input.R
@@ -174,10 +174,11 @@ create_model_input <-
       warning("Startingloc parameter is not specified. Rerun the create_alternative_choice function")
     }
 
+    # Flattened identity matrices (alts x alts) in each row, and nrows = number of observations
     dataCompile <- create_logit_input(choice)
 
     # IMPORTANT NOTE: Both choice possibilities AND distances are sorted/shifted even though the column names for distances are not shifted.
-    d <- shift_sort_x(dataCompile, choice, catch, distance, max(choice), ab)
+    d <- shift_sort_x(x = dataCompile, ch = choice, y = catch, distance = distance, alts = max(choice), ab = ab)
 
     # Data needs will vary by the likelihood function
     if (grepl("epm", fr)) {

diff --git a/R/data_parser.R b/R/data_parser.R
@@ -527,7 +527,6 @@ load_maindata <- function(dat, project, over_write = FALSE, compare = FALSE, y =
 
   dataset <- data_upload_helper(dat, "main")
 
-
   # coerce to tibble
   # TODO: customize column name check (case-insensitive)
   dataset <- tibble::as_tibble(dataset)
@@ -1042,21 +1041,19 @@ load_aux <- function(dat, aux, name, over_write = TRUE, project = NULL) {
   }
 }
 
-load_grid <- function(dat, grid, name, over_write = TRUE, project = NULL) {
+load_grid <- function(grid, name, project, over_write = TRUE) {
   #' Import, parse, and save gridded data to FishSET database
   #'
   #' Gridded data is data that varies by two dimensions. Column names must be zone 
   #' names. Load, parse, and save gridded data to FishSET database.  A project must 
   #' exist before running \code{load_grid()}. See \code{\link{load_maindata}}
   #' to create a new project. 
   #' 
-  #' @param dat Primary data containing information on hauls or trips. 
-  #'   Table in FishSET database contains the string 'MainDataTable'.
   #' @param grid File name, including path, of gridded data. 
   #' @param name Name gridded data should be saved as in FishSET database.
+  #' @param project String, name of project.
   #' @param over_write Logical, If TRUE, saves dat over previously saved data table 
   #'   in the FishSET database.
-  #' @param project String, name of project.
   #' @details Grid data is an optional data frame that contains a variable that 
   #'   varies by the map grid (ex. sea surface temperature, wind speed). Data can 
   #'   also vary by a second dimension (e.g., date/time). Both dimensions in the 
@@ -1083,7 +1080,6 @@ load_grid <- function(dat, grid, name, over_write = TRUE, project = NULL) {
   #' }
   #' 
 
-
   if (project_exists(project) == FALSE) {
 
     stop("Project '", project, "' does not exist. Check spelling or create a",
@@ -1096,13 +1092,19 @@ load_grid <- function(dat, grid, name, over_write = TRUE, project = NULL) {
 
   check <- TRUE
 
-  out <- data_pull(dat, project)
-  # Note: dataset is not used
-  dataset <- out$dataset
-  dat <- parse_data_name(dat, "main", project)
+  if(!is.data.frame(grid)){
+    grid <- as.data.frame(grid)
+  }
 
   grid <- data_upload_helper(grid, "grid")
 
+  # Coerce to tibble
+  if(any(is.na(names(grid)))){
+    i_rm <- which(is.na(names(grid)))
+    grid <- grid[,-i_rm]
+  }
+  grid <- tibble::as_tibble(grid)
+
   if (check == FALSE) { 
 
     warning("Grid table not saved.")
@@ -1135,8 +1137,7 @@ load_grid <- function(dat, grid, name, over_write = TRUE, project = NULL) {
 
       load_gridded_function <- list()
       load_gridded_function$functionID <- "load_grid"
-      load_gridded_function$args <- list(deparse_name(dat), deparse_name(grid), 
-                                         name, over_write, project)
+      load_gridded_function$args <- list(deparse_name(grid), name, project, over_write)
       log_call(project, load_gridded_function)
 
       message("Grid table saved to database.")

diff --git a/R/discretefish_subroutine.R b/R/discretefish_subroutine.R
@@ -322,9 +322,9 @@ discretefish_subroutine <- function(project,
       }
 
       # likelihood ----
-      LL_start <- fr.name(starts2, datamatrix$d, datamatrix$otherdat, 
-                          max(datamatrix$choice), project = project, 
-                          datamatrix$expname, as.character(datamatrix$mod.name))
+      LL_start <- fr.name(starts3 = starts2, dat = datamatrix$d, otherdat = datamatrix$otherdat, 
+                          alts = max(datamatrix$choice), project = project, 
+                          expname = datamatrix$expname, mod.name = as.character(datamatrix$mod.name))
 
       if (is.null(LL_start) || is.nan(LL_start) || is.infinite(LL_start)) {
         # haven't checked what happens when error yet
@@ -444,12 +444,12 @@ discretefish_subroutine <- function(project,
 
       PseudoR2 <- round((LL_start - LL) / LL_start, 3)
 
-      if (!is.null(datamatrix$expname)) {
-
-        modOutName <- paste0(c(mdf[[i]][["mod.name"]], datamatrix$expname), collapse = '.')
-        
-      } else modOutName <- mdf[[i]][["mod.name"]]
-      
+      # if (!is.null(datamatrix$expname)) {
+      #   
+      #   modOutName <- paste0(c(mdf[[i]][["mod.name"]], datamatrix$expname), collapse = '.')
+      # } else 
+      modOutName <- mdf[[i]][["mod.name"]]
+
       mod.out <- data.frame(matrix(NA, nrow = 4, ncol = 1))
       mod.out[, 1] <- c(AIC, AICc, BIC, PseudoR2)
       rownames(mod.out) <- c("AIC", "AICc", "BIC", "PseudoR2")
@@ -607,7 +607,7 @@ discretefish_subroutine <- function(project,
 
           rownames(OutLogit) <- c(n1, ind_vars, n2, n3)
 
-        }else {
+        } else {
           # Q: will this always be the correct order?
           rownames(OutLogit) <- c(ec_names, p_names)
         }

diff --git a/R/format_grid.R b/R/format_grid.R
@@ -58,7 +58,6 @@ format_grid <-
     grid <- parse_data_name(grid, "grid", project)
 
     # Note: if no date var, allow format?
-
     stopifnot("'area.dat' is required" = !is_value_empty(area.dat),
               "'id.cols' is required" = !is_value_empty(id.cols))
 
@@ -72,7 +71,7 @@ format_grid <-
       g_names <- colnames(griddat)
       g_areas <- g_names[!g_names %in% id.cols]
       d_areas <- unique(as.character(dataset[[area.dat]]))
-      a_ind <- g_areas %in% d_areas
+      a_ind <- g_names %in% d_areas
 
       if (sum(a_ind) == 0) {
 

diff --git a/R/logit_c.R b/R/logit_c.R
@@ -81,7 +81,6 @@ logit_c <- function(starts3, dat, otherdat, alts, project, expname, mod.name) {
   #' }
   #'
 
-
   griddat <- as.matrix(do.call(cbind, otherdat$griddat))
   intdat <- as.matrix(do.call(cbind, otherdat$intdat))
 
@@ -113,9 +112,9 @@ logit_c <- function(starts3, dat, otherdat, alts, project, expname, mod.name) {
 
   # Sum beta and gamma components of the model for each observation
   prof <- rowSums(djztemp, dims = 2)
-  profx <- prof - prof[, 1]
+  profx <- prof - prof[, 1] # relative to the alternative that was chosen
 
-  exb <- exp(profx)
+  exb <- exp(profx) # numerator of the conditional logit function
 
   ldchoice <- (-log(rowSums(exb)))
 

diff --git a/R/make_model_design.R b/R/make_model_design.R
@@ -290,13 +290,11 @@ make_model_design <-
 
     column_check(dataset, c(catchID, vars1, priceCol, startloc))
 
-    lapply(vars2, function(x) {
-
-      if (!table_exists(x, project)) {
-
-        stop("Gridded table '", x, "' does not exist.", call. = FALSE)
+    invisible(lapply(vars2, function(x){
+      if(!table_exists(paste0(project, x, "GridTable"), project)){
+        stop("Gridded table '", paste0(project,x,"GridTable"), "' does not exist.", call. = FALSE)
       }
-    })
+    }))
 
   } else {
 
@@ -417,7 +415,6 @@ make_model_design <-
     }
   }
 
-
   alt_var <- Alt$alt_var
   occasion <- Alt$occasion
   occasion_var <- Alt$occasion_var
@@ -567,10 +564,7 @@ make_model_design <-
   # }
 
   # Gridded ----
-  # Note: create_alternative_choice() currently cannot create a distance matrix from a 
   # gridded dataset
-
-
   if (is_value_empty(gridVars)) {
 
     if (is_value_empty(unlist(expectcatchmodels))) {
@@ -588,12 +582,35 @@ make_model_design <-
       # TODO: check if gridded table has correct # of rows, if not error out and 
       # tell user to re-run format_grid()
       gridVariablesInclude <- lapply(gridVars, function(x) {
+
+        grid_tab <- table_view(paste0(project, x, "GridTable"), project)
+
+        if(dim(grid_tab)[1] > 1) {
+          stop("The two dimensional alternative-specfic variable option is under development. 
+               Use single dimensional grid variables. Please check with developers for updates on progress.")
+        }
 
-        grid_tab <- table_view(x, project)
+        if(!all(zoneRow$ZoneID %in% names(grid_tab))){
+          stop("One or more zones in the model are missing from the ",  x, " GridTable")
+        }
 
-        grid_tab[zone_ind, names(grid_tab) %in% unique(choice)]
+        grid_tab
       })
 
+      # Change format to match expected catch dimensions
+      gridVariablesInclude <- lapply(gridVariablesInclude, function(x) {
+        grid_tab <- x[,which(names(x) %in% zoneRow$ZoneID)] # Get only the zones in the model
+        grid_tab <- do.call("rbind", replicate(sum(dataZoneTrue), grid_tab, simplify = FALSE))
+      })
+
+      check_dim <- lapply(gridVariablesInclude, function(x) {
+        all(dim(x) == dim(ExpectedCatch[[1]]))
+      })
+
+      if(!all(unlist(check_dim))){
+        stop("Dimensions of grid variables do not match expected catch matrix dimensions. Check grid tables")
+      }
+
     } else {
 
       gridVariablesInclude <- lapply(gridVars, function(x) dataset[[x]][zone_ind])

diff --git a/R/welfare_predict.R b/R/welfare_predict.R
@@ -45,7 +45,6 @@ welfare_predict <- function(project, mod.name, closures, betadraws = 1000, marg_
   fishset_db <- DBI::dbConnect(RSQLite::SQLite(), locdatabase(project=project))
   on.exit(DBI::dbDisconnect(fishset_db), add = TRUE)
 
-
   #---
   # Pull output data from model optimization ----
   #---
@@ -130,7 +129,6 @@ welfare_predict <- function(project, mod.name, closures, betadraws = 1000, marg_
     } 
   } 
 
-
   #---
   # Save coefficients ----
   #---
@@ -295,7 +293,6 @@ welfare_predict <- function(project, mod.name, closures, betadraws = 1000, marg_
         # Check if theta is negative value
         if(!isRunning()){
           if(theta < 0) stop("Marginal utility of income is negative. Check model coefficient (estimate and standard error) and select appropriate marginal utility of income.")
-
         } 
 
         # set up distance matrix
@@ -312,8 +309,6 @@ welfare_predict <- function(project, mod.name, closures, betadraws = 1000, marg_
           dim(gridbetas) <- c(nrow(gridbetas), alts, gridnum)
           gridbetas <- rowSums(gridbetas, dims = 2)
 
-          as.vector(matrix(rep(gridcoef, each = alts), nrow = obsnum, ncol = alts*2, byrow = TRUE))
-
           ## logit_zonal ----
         } else if(mod.ll == "logit_zonal"){
           # insert 0 for the first alternative (interpretation is relative to the first alt and this beta_1 = 0)

diff --git a/inst/ShinyFiles/MainApp/run_policy_server.R b/inst/ShinyFiles/MainApp/run_policy_server.R
@@ -23,7 +23,7 @@ pred_plotsServer <- function(id, project, spatdat, values){
         req(project)
 
         selectInput(ns("select_pol_mod"), 'Select a model', 
-                    choices = c(model_names(project)))
+                    choices = c(lapply(model_out_view(project=project), "[[", "name")))
 
       })
 
@@ -50,11 +50,12 @@ pred_plotsServer <- function(id, project, spatdat, values){
       output$pol_likelihood <- renderUI({
         req(project)
         req(input$select_pol_mod)
+
         if((model_design_list(project)[[which(lapply(model_design_list(project=project), "[[", "mod.name")
-                                                      == input$select_pol_mod)]]$likelihood %in% c("logit_c", "logit_zonal"))){
+                                              == input$select_pol_mod)]]$likelihood %in% c("logit_c", "logit_zonal"))){
           tagList(
           selectInput(ns("select_marg_inc"),'Marginal utility of income coefficient', 
-                     choices = row.names(model_out_view(project)[[which(lapply(model_design_list(project=project), "[[", "mod.name")
+                     choices = row.names(model_out_view(project)[[which(lapply(model_out_view(project), "[[", "name")
                                                                           == input$select_pol_mod)]]$OutLogit)),
 
           add_prompter(