adding species notes, spaghetti plots, and model commenting tweaks

R/aa_common_docs.R

@@ -31,7 +31,16 @@
 #'   quantiles need to have been precalculated in `generate_indices()`.
 #' @param add_observed_means Logical. Whether to include points indicating the
 #'   observed mean counts. Default `FALSE`. Note: scale of observed means and
-#'   annual indices may not match due to imbalanced sampling among routes.
+#'   annual indices may not match due to imbalanced sampling among routes. Also,
+#'   pattern of change in observed means through time should reflect estimated
+#'   population trajectories for strata-level summaries. However, the temporal
+#'   patterns may be very different between means and annual indices for composite
+#'   regions (e.g., continental, state, province, or BCR) because the indices
+#'   for composite regions account for the variation in area weights and
+#'   variation in relative abundances among strata, and the observed mean counts
+#'   do not. For strata-level assessments, these observed means can be a useful
+#'   model assessment tool. For composite regions, their interpretation is more
+#'   complicated.
 #' @param col_viridis Logical. Should the colour-blind-friendly "viridis"
 #'   palette be used. Default `FALSE`.
 #'
@@ -48,7 +57,7 @@
 #'   better descriptor of skewed posterior distributions, such as the predicted
 #'   mean counts that the indices represent.
 #'   Note hpd intervals are not stable for small percentages of the posterior
-#'   distribution, and so `hdpi = TRUE` is ignored for `quantiles` values
+#'   distribution, and so `hpdi = TRUE` is ignored for `quantiles` values
 #'   between 0.33 and 0.67 (i.e., if the `quantiles` value defines a limit for
 #'   a centered hpd interval that would include < 33% of the
 #'   posterior distribution).

R/fetch-bbs-data.R

@@ -30,7 +30,7 @@
 #' individually. **Note that stop-level data is not currently supported by the
 #' modelling utilities in bbsBayes2.**
 #'
-#' There are three releases for each type of data, `2020`, `2022`, `2023`,
+#' There are multiple releases for each type of data, `2020`, `2022`, `2023`,
 #' and `2024`.
 #' By default all functions use the most recent release unless otherwise
 #' specified. For example, the `release` argument in `stratify()` can be

R/generate-indices.R

@@ -257,7 +257,7 @@ generate_indices <- function(
       dplyr::mutate(
         pz_area = .data$area_sq_km * .data$non_zero_weight,
         strata_p = .data$pz_area / sum(.data$pz_area),
-        area_weight = .data$area_sq_km / sum(.data$area_sq_km),
+        area_weight = .data$area_sq_km / sum(.data$area_sq_km,na.rm = TRUE),
         area_weight_non_zero = .data$area_weight * .data$non_zero_weight) %>%
       dplyr::ungroup()
 

R/generate-trends.R

@@ -192,7 +192,7 @@ generate_trends <- function(indices,
     dplyr::summarize(
       # Basic statistics
       rel_abundance = mean(.data$index),
-      obs_rel_abundance = mean(.data$obs_mean),
+      obs_rel_abundance = mean(.data$obs_mean,na.rm = TRUE),
       mean_n_routes = mean(.data$n_routes),
       n_routes = mean(.data$n_routes_total),
       backcast_flag = mean(.data$backcast_flag),

R/plot-indices.R

@@ -13,7 +13,17 @@
 #'   showing the number of BBS routes included in each year. This is useful as a
 #'   visual check on the relative data-density through time because in most
 #'   cases the number of observations increases over time.
-#'
+#' @param spaghetti Logical. False by default. Plotting option to visualise the
+#'   uncertainty in the estimated population trajectories. Instead of plotting
+#'   the trajectory as a single line with an uncertainty bound, if TRUE, then
+#'   the plot shows a sample of the posterior distribution of the estimated
+#'   population trajectories. E.g., 100 semi-transparent lines, each representing
+#'   one posterior draw of the population trajectory.
+#' @param n_spaghetti Integer. 100 by default. Number of posterior draws of the
+#'   population trajectory to include in the plot. Ignored if spaghetti = FALSE.
+#' @param alpha_spaghetti Numeric [0,1], 0.2 by default. Alpha value - transparency
+#'   of each individual population trajectory line in the spaghetti plot.
+#'   Ignored if spaghetti = FALSE.
 #' @inheritParams common_docs
 #' @family indices and trends functions
 #'
@@ -63,20 +73,32 @@ plot_indices <- function(indices = NULL,
                          axis_title_size = 18,
                          axis_text_size = 16,
                          line_width = 1,
+                         spaghetti = FALSE,
+                         n_spaghetti = 100,
+                         alpha_spaghetti = 0.2,
                          add_observed_means = FALSE,
                          add_number_routes = FALSE) {
 
   # Checks
   check_data(indices)
   check_logical(title, add_observed_means, add_number_routes)
   check_numeric(ci_width, title_size, axis_title_size, axis_text_size, line_width)
+  check_numeric(alpha_spaghetti,n_spaghetti)
+
   check_numeric(min_year, max_year, allow_null = TRUE)
   check_range(ci_width, c(0.001, 0.999))
 
+  if(spaghetti){line_width = line_width*0.1}
+
   species <- indices$meta_data$species
+
+  indices_samples <- indices$samples
+
   indices <- indices$indices %>%
     calc_luq(ci_width)
 
+
+
   cl <- "#39568c"
 
   plot_list <- list()
@@ -85,21 +107,37 @@ plot_indices <- function(indices = NULL,
   if(!is.null(max_year)) indices <- indices[indices$year <= max_year, ]
 
   for (i in unique(indices$region)) {
-
     to_plot <- indices[which(indices$region == i), ]
+
+    samples_name <- to_plot %>%
+      dplyr::select(region_type,region) %>%
+      dplyr::distinct() %>%
+      as.character() %>%
+      paste(collapse = "_")
+
+    to_plot_spaghetti <- indices_samples[[samples_name]] %>%
+      as.data.frame() %>%
+      dplyr::mutate(iteration = dplyr::row_number()) %>%
+      dplyr::sample_n(size = n_spaghetti) %>%
+      tidyr::pivot_longer(cols = !dplyr::matches("iteration"),
+                          values_to = "index",
+                          names_to = "year") %>%
+      dplyr::mutate(year = as.integer(.data$year))
+
+
     if(title) t <- paste0(species, " - ", i) else t <- ""
 
     if(add_number_routes){
       if(max(to_plot$n_routes) > 200) {
         ncby_y <- ceiling(to_plot$n_routes/50)
-        annot <- c("each dot ~ 50 routes")
+        annot <- c("1 dot ~ 50 routes")
       } else {
         if(max(to_plot$n_routes) > 100) {
           ncby_y <- ceiling(to_plot$n_routes/10)
-          annot <- c("each dot ~ 10 routes")
+          annot <- c("1 dot ~ 10 routes")
         } else {
           ncby_y <- to_plot$n_routes
-          annot <- c("each dot = 1 route")
+          annot <- c("1 dot = 1 route")
         }
       }
 
@@ -131,15 +169,23 @@ plot_indices <- function(indices = NULL,
                             colour = "grey60", na.rm = TRUE)
     }
 
+    if(spaghetti){
     p <- p +
       ggplot2::geom_line(
-        data = to_plot, ggplot2::aes(x = .data$year, y = .data$index),
-        colour = cl, linewidth = line_width, ) +
-      ggplot2::geom_ribbon(
-        data = to_plot,
-        ggplot2::aes(x = .data$year, ymin = .data$lci, ymax = .data$uci),
-        fill = cl, alpha = 0.3)
-
+        data = to_plot_spaghetti, ggplot2::aes(x = .data$year, y = .data$index,
+                                               group = .data$iteration),
+        colour = cl, linewidth = line_width,
+        alpha = alpha_spaghetti)
+    }else{
+      p <- p +
+        ggplot2::geom_line(
+          data = to_plot, ggplot2::aes(x = .data$year, y = .data$index),
+          colour = cl, linewidth = line_width, ) +
+        ggplot2::geom_ribbon(
+          data = to_plot,
+          ggplot2::aes(x = .data$year, ymin = .data$lci, ymax = .data$uci),
+          fill = cl, alpha = 0.3)
+    }
     if(add_number_routes) {
       p <- p +
         ggplot2::geom_dotplot(

R/prepare-data.R

@@ -75,6 +75,35 @@ prepare_data <- function(strata_data,
     dplyr::left_join(obs, by = c("route", "rpid", "year")) %>%
     dplyr::mutate(count = tidyr::replace_na(.data$count, 0))
 
+  # Warn if min_year is null and species is a lumped species complex
+  # of if species has expanded into BBS range
+  if(strata_data$meta_data$sp_aou %in% bbsBayes2::species_notes$aou){
+    sp_note <- species_notes %>%
+      dplyr::filter(.data$aou == .env$strata_data$meta_data$sp_aou)
+      sp_note_y <- sp_note$minimum_year
+      sp_note_msg <- sp_note$warning
+
+      if(is.null(min_year)){
+        warning(paste("Consider rerunning prepare_data(min_year = ",
+                      sp_note_y,") or some later year.",
+                sp_note_msg),
+                call. = FALSE, immediate. = TRUE)
+      }else{
+
+      if(min_year < sp_note_y){
+        warning(paste("Consider rerunning prepare_data(min_year = ",
+                      sp_note_y,") or some later year.",
+                      sp_note_msg),
+                call. = FALSE, immediate. = TRUE)
+      }
+      }
+
+    }else{
+      sp_note_y <- NULL
+      sp_note_msg <- NULL
+    }
+
+
   # Filter observations
   if(!is.null(min_year)) obs <- dplyr::filter(obs, .data$year >= .env$min_year)
   if(!is.null(max_year)) obs <- dplyr::filter(obs, .data$year <= .env$max_year)

R/run-model.R

@@ -39,6 +39,12 @@
 #'   the model output file save object, when `save_model = TRUE`, and so for
 #'   file organization and efficient use of memory, these are deleted
 #'   by default.
+#' @param show_exceptions Logical.  Passed to `cmdstanr::sample()`.
+#'   Defaults to FALSE. When TRUE, prints all informational messages from Stan,
+#'   for example rejection of the current proposal. Disabled by default in
+#'   bbsBayes2, because of the copious informational messages during the
+#'   initialization period that have no bearing on model fit. If fitting a
+#'   custom model, recommend setting this to TRUE.
 #' @param ... Other arguments passed on to `cmdstanr::sample()`.
 #'
 #' @inheritParams common_docs
@@ -86,6 +92,7 @@ run_model <- function(model_data,
                       retain_csv = FALSE,
                       set_seed = NULL,
                       quiet = FALSE,
+                      show_exceptions = FALSE,
                       ...) {
 
   # Check inputs
@@ -180,6 +187,7 @@ run_model <- function(model_data,
     init = init_values,
     output_dir = output_dir,
     output_basename = output_basename,
+    show_exceptions = show_exceptions,
     ...)
 
   model_output <- list("model_fit" = model_fit,

R/stratify.R

@@ -308,7 +308,8 @@ stratify <- function(by,
   # Return list
   out <- list("meta_data" = list("stratify_by" = stratify_by,
                                  "stratify_type" = stratify_type,
-                                 "species" = species),
+                                 "species" = species,
+                                 "sp_aou" = sp_aou),
               "meta_strata" = meta_strata,
               "birds_strata" = dplyr::select(birds, -"rid"),
               "routes_strata" = dplyr::select(routes, -"rid"))

data-raw/data_exported.R

@@ -15,6 +15,14 @@ species_forms <- readr::read_csv("data-raw/species_forms.csv",
 usethis::use_data(species_forms, overwrite = TRUE)
 
 
+species_notes <- readr::read_csv("data-raw/species_notes.csv",
+                                 col_types = "ncc", locale = readr::locale(encoding = "latin1"),
+                                 progress = FALSE) %>%
+  as.data.frame()
+
+usethis::use_data(species_notes, overwrite = TRUE)
+
+
 bbs_models <- dplyr::tribble(
   ~model,       ~variant,           ~file,
   "first_diff", "nonhier", "first_diff_nonhier_bbs_CV.stan",

data-raw/species_forms.csv

@@ -11,4 +11,5 @@ aou_original,english_original,english_out,french_out,n_add,aou1,aou2,aou3,aou4,a
 4642,unid. Cordilleran / Pacific-slope Flycatcher,Western Flycatcher (Cordilleran/Pacific-slope),Moucherolle c�tier (des ravins/ c�tier),2,4641,4640,,,
 12,unid. Western Grebe / Clark's Grebe,Western Grebe (Clark's/Western),Gr�be �l�gant (� face blanche/�l�gant),2,10,11,,,
 6556,(unid. Myrtle/Audubon's) Yellow-rumped Warbler,Yellow-rumped Warbler (all forms),Paruline � croupion jaune (toutes les formes),2,6550,6560,,,
-5275,unid. Common Redpoll / Hoary Redpoll,Redpoll (Common/Hoary),Sizerin (Flamm�/Blanch�tre),2,5270,5280,,,
+5275,unid. Common Redpoll / Hoary Redpoll,Redpoll (Common/Hoary),Sizerin (flamm�/blanch�tre),2,5270,5280,,,
+5012,unid. Meadowlark,Meadowlark (Eastern/Western/Chihuahuan),Sturnelle (pr�s/Ouest/Lilian),3,5009,5010,5011,,

data-raw/species_notes.csv

@@ -0,0 +1,7 @@
+aou,minimum_year,warning
+4661,1978,Alder and Willow Flycatcher were considered a single species until 1973. It is likely that they are not accurately separated by BBS observers until at least some years after that split.
+4660,1978,Alder and Willow Flycatcher were considered a single species until 1973. It is likely that they are not accurately separated by BBS observers until at least some years after that split.
+10,1990,Clark's and Western Grebe were considered a single species until 1985. It is likely that they are not accurately separated by BBS observers until at least some years after that split.
+10,1990,Clark's and Western Grebe were considered a single species until 1985. It is likely that they are not accurately separated by BBS observers until at least some years after that split.
+22860,1990,Eurasian Collared Dove was introduced into North America in the 1980s. 1990 is the first year that the species was observed on at least 3 BBS routes. 
+6121,1985,Cave Swallows were relatively rare in the areas surveyed by BBS before 1980. There are only two observations during BBS before 1980.

inst/models/gam_hier_bbs_CV.stan

@@ -5,19 +5,6 @@
 // facilitate the ragged array issues and to reduce the model output size (greatly)
 // althought nice to have them here where Rhat and ess_ are calculated
 
-// iCAR function, from Morris et al. 2019
-// Morris, M., K. Wheeler-Martin, D. Simpson, S. J. Mooney, A. Gelman, and C. DiMaggio (2019).
-// Bayesian hierarchical spatial models: Implementing the Besag York Mollié model in stan.
-// Spatial and Spatio-temporal Epidemiology 31:100301.
-
- functions {
-   real icar_normal_lpdf(vector bb, int ns, array[] int n1, array[] int n2) {
-     return -0.5 * dot_self(bb[n1] - bb[n2])
-       + normal_lpdf(sum(bb) | 0, 0.001 * ns); //soft sum to zero constraint on bb
-  }
- }
-
-
 
 data {
   int<lower=1> n_sites;

inst/models/gamye_hier_bbs_CV.stan

@@ -4,19 +4,6 @@
 // facilitate the ragged array issues and to reduce the model output size (greatly)
 // althought nice to have them here where Rhat and ess_ are calculated
 
-// iCAR function, from Morris et al. 2019
-// Morris, M., K. Wheeler-Martin, D. Simpson, S. J. Mooney, A. Gelman, and C. DiMaggio (2019).
-// Bayesian hierarchical spatial models: Implementing the Besag York Mollié model in stan.
-// Spatial and Spatio-temporal Epidemiology 31:100301.
-
- functions {
-   real icar_normal_lpdf(vector bb, int ns, array[] int n1, array[] int n2) {
-     return -0.5 * dot_self(bb[n1] - bb[n2])
-       + normal_lpdf(sum(bb) | 0, 0.001 * ns); //soft sum to zero constraint on bb
-  }
- }
-
-
 
 data {
   int<lower=1> n_sites;