From 9d60a4b9b55ae12ff0dbe0e743cd56ee5b6a4256 Mon Sep 17 00:00:00 2001 From: Cory McCartan Date: Wed, 22 Apr 2020 21:54:52 -0400 Subject: [PATCH] core code complete --- NAMESPACE | 2 + R/adjust_weights.R | 2 + R/adjustr-package.R | 1 + R/logprob.R | 2 +- R/mockup.R | 1 + R/parsing.R | 4 +- R/use_weights.R | 90 +++++++ README.md | 10 +- _pkgdown.yml | 9 +- docs/404.html | 2 +- docs/LICENSE-text.html | 2 +- docs/authors.html | 2 +- docs/index.html | 6 +- docs/pkgdown.yml | 2 +- docs/reference/adjust_weights.html | 8 +- docs/reference/adjustr.html | 7 +- .../reference/as.data.frame.adjustr_spec.html | 2 +- docs/reference/dplyr.adjustr_spec.html | 2 +- docs/reference/extract_samp_stmts.html | 2 +- docs/reference/get_resampling_idxs.html | 2 +- docs/reference/index.html | 27 +- docs/reference/make_spec.html | 6 +- docs/reference/pull.adjustr_weighted.html | 2 +- ...c.html => summarise.adjustr_weighted.html} | 61 ++--- .../reference/summarize.adjustr_weighted.html | 245 ++++++++++++++++++ docs/sitemap.xml | 3 + inst/stan.g | 2 +- man/adjustr.Rd | 1 + man/summarize.adjustr_weighted.Rd | 38 +++ tests/testthat/test_use.R | 63 ++++- tests/testthat/test_weights.R | 1 + 31 files changed, 522 insertions(+), 85 deletions(-) rename docs/reference/{filter.adjust_spec.html => summarise.adjustr_weighted.html} (71%) create mode 100644 docs/reference/summarize.adjustr_weighted.html create mode 100644 man/summarize.adjustr_weighted.Rd diff --git a/NAMESPACE b/NAMESPACE index 8ecc8db..e0d5d6f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -8,6 +8,8 @@ S3method(pull,adjustr_weighted) S3method(rename,adjustr_spec) S3method(select,adjustr_spec) S3method(slice,adjustr_spec) +S3method(summarise,adjustr_weighted) +S3method(summarize,adjustr_weighted) export(adjust_weights) export(extract_samp_stmts) export(get_resampling_idxs) diff --git a/R/adjust_weights.R b/R/adjust_weights.R index 9bdab55..0028482 100644 --- a/R/adjust_weights.R +++ b/R/adjust_weights.R @@ -76,6 +76,8 @@ adjust_weights = function(spec, object, data=NULL, keep_bad=F) { if (!keep_bad) adjust_obj$.weights[adjust_obj$.pareto_k > 0.7] = list(NA_real_) attr(adjust_obj, "draws") = rstan::extract(object) + attr(adjust_obj, "data") = data + attr(adjust_obj, "iter") = object@sim$chains * (object@sim$iter - object@sim$warmup) adjust_obj } diff --git a/R/adjustr-package.R b/R/adjustr-package.R index 663bf2b..dedaf24 100644 --- a/R/adjustr-package.R +++ b/R/adjustr-package.R @@ -15,6 +15,7 @@ #' \itemize{ #' \item \code{\link{make_spec}} #' \item \code{\link{adjust_weights}} +#' \item \code{\link{summarize}} #' } #' #' @importFrom methods is diff --git a/R/logprob.R b/R/logprob.R index 069293c..797866d 100644 --- a/R/logprob.R +++ b/R/logprob.R @@ -126,4 +126,4 @@ if (requireNamespace("extraDistr", quietly=T)) { } # Turn mapping into an environment suitable for metaprogramming, # and turn each density into its curried form (see `make_dens` above) -distr_env = new_environment(purrr::map(distrs, make_dens), parent=empty_env()) +distr_env = new_environment(purrr::map(distrs, make_dens)) diff --git a/R/mockup.R b/R/mockup.R index 6c866f8..187eca3 100644 --- a/R/mockup.R +++ b/R/mockup.R @@ -32,6 +32,7 @@ eightschools_m = stan(model_code=model_code, chains=2, data=model_d, warmup=500, iter=510, save_dso=F, save_warmup=F) eightschools_m@stanmodel@dso = new("cxxdso") save(eightschools_m, file="tests/test_model.rda") +load("tests/test_model.rda") #slot(eightschools_m@stanmodel, "dso", F) = NULL #draws = extract(eightschools_m) diff --git a/R/parsing.R b/R/parsing.R index 453b565..13ecd36 100644 --- a/R/parsing.R +++ b/R/parsing.R @@ -10,11 +10,11 @@ get_parser = function() { # nocov start } } # nocov end -# Parse Satan `model_code` into a data frame which represents the parsing tree +# Parse Stan `model_code` into a data frame which represents the parsing tree parse_model = function(model_code) { parser_output = utils::capture.output( get_parser()(model_code, function(name, value, pos, depth) { - cat(stringr::str_glue('"{name}","{value}",{pos},{depth}\n\n')) + cat('"', name, '","', value, '",', pos, ',', depth, '\n', sep="") }) ) parser_csv = paste0("name,value,pos,depth\n", diff --git a/R/use_weights.R b/R/use_weights.R index e026e27..fa673c0 100644 --- a/R/use_weights.R +++ b/R/use_weights.R @@ -34,3 +34,93 @@ get_resampling_idxs = function(x, frac=1, replace=T) { get_idxs(x) } } + +#' Summarize Posterior Distributions Under Alternative Model Specifications +#' +#' Uses weights computed in \code{\link{adjust_weights}} to compute posterior +#' summary statistics. These statistics can be compared against their reference +#' values to quantify the sensitivity of the model to aspects of its +#' specification. +#' +#' @param .data An \code{adjustr_weighted} object. +#' @param ... Name-value pairs of expressions. The name of each argument will be +#' the name of a new variable, and the value will be computed for the +#' posterior distribution of eight alternative specification. For example, +#' a value of \code{mean(theta)} will compute the posterior mean of +#' \code{theta} for each alternative specification. +#' @param .resampling Wether to compute summary statistics by first resampling +#' the data according to the weights. Defaults to \code{FALSE}, but will be +#' used for any summary statistic that is not \code{mean}, \code{var} or +#' \code{sd}. +#' @param .model_data Stan model data, if not provided in the earlier call to +#' \code{\link{adjust_weights}}. +#' +#' @return An \code{adjustr_weighted} object, wth the new columns specified in +#' \code{...} added. +#' +#' @rdname summarize.adjustr_weighted +#' @export +summarise.adjustr_weighted = function(.data, ..., .resampling=F, .model_data=NULL) { + stopifnot(is.adjustr_weighted(.data)) # just in case called manually + args = enexprs(...) + + broadcast = function(x) { + dims = c(dim(as.array(x)), iter) + x = array(rep(x, iter), dim=dims) + aperm(x, c(length(dims), 2:length(dims) - 1)) + } + iter = attr(.data, "iter") + if (!is_null(.model_data)) attr(.data, "data") = .model_data + data = append(attr(.data, "draws"), map(attr(.data, "data"), broadcast)) + + n_args = length(args) + for (i in seq_along(args)) { + name = names(args)[i] + if (name == "") name = expr_name(args[[i]]) + + call = args[[i]] + if (!.resampling && exists(call_name(call), funs_env)) { + fun = call_fn(call, funs_env) + } else { + fun = function(x, ...) apply(x, 2, call_fn(call), ...) + .resampling = T + } + + expr = expr_deparse(call_args(call)[[1]]) + expr = stringr::str_replace_all(expr, "\\[(\\d)", "[,\\1") + expr = stringr::str_replace_all(expr, "(? Various helper functions for examining a model or building sampling specifications. contents: - extract_samp_stmts + - as.data.frame.adjustr_spec - dplyr.adjustr_spec - get_resampling_idxs - pull.adjustr_weighted - - title: "Plotting Functions" - desc: > - Functions to visualize changes to a model's posterior across - specifications. - contents: diff --git a/docs/404.html b/docs/404.html index 7b0fc17..9ba785a 100644 --- a/docs/404.html +++ b/docs/404.html @@ -135,7 +135,7 @@
  • - + diff --git a/docs/LICENSE-text.html b/docs/LICENSE-text.html index 4ed1c8c..0909ec3 100644 --- a/docs/LICENSE-text.html +++ b/docs/LICENSE-text.html @@ -135,7 +135,7 @@
  • - + diff --git a/docs/authors.html b/docs/authors.html index 6522fc6..e531682 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -135,7 +135,7 @@
  • - + diff --git a/docs/index.html b/docs/index.html index 6ad08a3..137bb7c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -92,7 +92,7 @@
  • - + @@ -120,9 +120,9 @@ adjustr -

    adjustr is an R package which provides functions to help assess the sensitivity of a Bayesian model (fitted with Stan) to the specification of its likelihood and priors. Users provide a series of alternate sampling specifications, and the package uses Pareto-smoothed importance sampling to estimate posterior quantities of interest under each specification. The package also provides functions to summarize and plot how these quantities change across specifications.

    +

    adjustr is an R package which provides functions to help assess the sensitivity of a Bayesian model (fitted with Stan) to the specification of its likelihood and priors. Users provide a series of alternate sampling specifications, and the package uses Pareto-smoothed importance sampling to estimate the posterior under each specification. The package also provides functions to summarize and plot how posterior quantities quantities change across specifications.

    The package aims to provide simple interface that makes it as easy as possible for modellers to try out various adjustments to their Stan models, without needing to write any specific Stan code or even recompile or rerun their model.

    -

    The package works by parsing Stan model code, so everything works best if the model was written by the user. The complexity of rstanarm and brms models makes it harder to make interpretable adjustments, but in principle these are possible too.

    +

    The package works by parsing Stan model code, so everything works best if the model was written by the user. Models made using brms may in principle be used as well. Models made using rstanarm are constructed using more complex model templates, and cannot be used.

    Getting Started

    diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 3d8dc78..590fbf2 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -2,7 +2,7 @@ pandoc: 2.3.1 pkgdown: 1.5.1 pkgdown_sha: ~ articles: [] -last_built: 2020-04-19T01:34Z +last_built: 2020-04-23T01:51Z urls: reference: https://corymccartan.github.io/adjustr//reference article: https://corymccartan.github.io/adjustr//articles diff --git a/docs/reference/adjust_weights.html b/docs/reference/adjust_weights.html index d62cea1..8523fff 100644 --- a/docs/reference/adjust_weights.html +++ b/docs/reference/adjust_weights.html @@ -140,7 +140,7 @@
  • - + @@ -184,7 +184,7 @@

    Arg spec

    An object of class adjustr_spec, probably produced by -make_spec, containing the new sampling sampling statements +make_spec, containing the new sampling sampling statements to replace their counterparts in the original Stan model, and the data, if any, by which these sampling statements are parametrized.

    @@ -211,11 +211,11 @@

    Arg

    Value

    A tibble, produced by converting the provided specs to a - tibble (see as.data.frame.adjustr_spec), and adding columns + tibble (see as.data.frame.adjustr_spec), and adding columns .weights, containing vectors of weights for each draw, and .pareto_k, containing the diagnostic Pareto shape parameters. Values greater than 0.7 indicate that importance sampling is not reliable. - Weights can be extracted with the pull.adjustr_weighted + Weights can be extracted with the pull.adjustr_weighted method. The returned object also includes the model sample draws, in the draws attribute.

    diff --git a/docs/reference/adjustr.html b/docs/reference/adjustr.html index cc351c1..02c70d6 100644 --- a/docs/reference/adjustr.html +++ b/docs/reference/adjustr.html @@ -142,7 +142,7 @@
  • - + @@ -191,8 +191,9 @@

    -

  • make_spec

    • -

  • adjust_weights

    • +

  • make_spec

    • +

  • adjust_weights

    • +

  • summarize

    • diff --git a/docs/reference/as.data.frame.adjustr_spec.html b/docs/reference/as.data.frame.adjustr_spec.html index 2e77ff6..2cb01f0 100644 --- a/docs/reference/as.data.frame.adjustr_spec.html +++ b/docs/reference/as.data.frame.adjustr_spec.html @@ -138,7 +138,7 @@

  • - + diff --git a/docs/reference/dplyr.adjustr_spec.html b/docs/reference/dplyr.adjustr_spec.html index 5875d73..b85131f 100644 --- a/docs/reference/dplyr.adjustr_spec.html +++ b/docs/reference/dplyr.adjustr_spec.html @@ -140,7 +140,7 @@
  • - + diff --git a/docs/reference/extract_samp_stmts.html b/docs/reference/extract_samp_stmts.html index 94a27ff..ddb74d4 100644 --- a/docs/reference/extract_samp_stmts.html +++ b/docs/reference/extract_samp_stmts.html @@ -138,7 +138,7 @@
  • - + diff --git a/docs/reference/get_resampling_idxs.html b/docs/reference/get_resampling_idxs.html index 3bf9b26..66278c4 100644 --- a/docs/reference/get_resampling_idxs.html +++ b/docs/reference/get_resampling_idxs.html @@ -137,7 +137,7 @@
  • - + diff --git a/docs/reference/index.html b/docs/reference/index.html index c9fac16..b53bd56 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -135,7 +135,7 @@
  • - + @@ -195,6 +195,12 @@

    summarise(<adjustr_weighted>) summarize(<adjustr_weighted>)

    + +

    Summarize Posterior Distributions Under Alternative Model Specifications

    @@ -216,6 +222,12 @@

    as.data.frame(<adjustr_spec>)

    + +

    Convert an adjustr_spec Object Into a Data Frame

    + +

    filter(<adjustr_spec>) arrange(<adjustr_spec>) rename(<adjustr_spec>) select(<adjustr_spec>) slice(<adjustr_spec>)

    @@ -233,19 +245,6 @@

    -

    Plotting Functions

    -

    Functions to visualize changes to a model’s posterior across specifications.

    - - - - - - - - diff --git a/docs/reference/make_spec.html b/docs/reference/make_spec.html index 68770e1..71d4558 100644 --- a/docs/reference/make_spec.html +++ b/docs/reference/make_spec.html @@ -138,7 +138,7 @@

  • - + @@ -170,7 +170,7 @@

    Set Up Model Adjustment Specifications

    Takes a set of new sampling statements, which can be parametrized by other arguments, data frames, or lists, and creates an adjustr_spec object -suitable for use in adjust_weights.

    +suitable for use in adjust_weights.

    make_spec(...)
    @@ -212,7 +212,7 @@

    Value

    An object of class adjustr_spec, which is essentially a list with two elements: samp, which is a list of sampling formulas, and params, which is a list of lists of parameters. Core -dplyr verbs which don't involve grouping +dplyr verbs which don't involve grouping (filter, arrange, mutate, select, rename, and slice) are diff --git a/docs/reference/pull.adjustr_weighted.html b/docs/reference/pull.adjustr_weighted.html index 8b147c6..60a1d8e 100644 --- a/docs/reference/pull.adjustr_weighted.html +++ b/docs/reference/pull.adjustr_weighted.html @@ -137,7 +137,7 @@

  • - + diff --git a/docs/reference/filter.adjust_spec.html b/docs/reference/summarise.adjustr_weighted.html similarity index 71% rename from docs/reference/filter.adjust_spec.html rename to docs/reference/summarise.adjustr_weighted.html index 982958f..1f84d68 100644 --- a/docs/reference/filter.adjust_spec.html +++ b/docs/reference/summarise.adjustr_weighted.html @@ -6,7 +6,7 @@ -<code>dplyr</code> methods for <code>adjustr_spec</code> objects — filter.adjustr_spec • adjustr +Summarize Posterior Distributions Under Alternative Model Specifications — summarise.adjustr_weighted • adjustr @@ -39,12 +39,11 @@ - - + + @@ -140,7 +139,7 @@
  • - + @@ -164,51 +163,47 @@
    -

    dplyr methods for adjustr_spec objects

    +

    Summarize Posterior Distributions Under Alternative Model Specifications

    - +
    -

    Core dplyr verbs which don't involve grouping - (filter, arrange, - mutate, select, - rename, and slice) are -implemented and operate on the underlying table of specification parameters.

    +

    Uses weights computed in adjust_weights to compute posterior +summary statistics. These statistics can be compared against their reference +values to quantify the sensitivity of the model to aspects of its +specification.

    - 
    # S3 method for adjustr_spec
-filter(.data, ..., .preserve = F)
-
-# S3 method for adjustr_spec
-arrange(.data, ...)
-
-# S3 method for adjustr_spec
-rename(.data, ...)
-
-# S3 method for adjustr_spec
-select(.data, ...)
-
-# S3 method for adjustr_spec
-slice(.data, ..., .preserve = F)
    + 
    # S3 method for adjustr_weighted
+summarise(.data, ..., .model_data = NULL)

    Arguments

    - + - + - - + +
    .data

    the adjustr_spec object

    An adjustr_weighted object.
    ...

    additional arguments to underlying method

    Name-value pairs of expressions. The name of each argument will be +the name of a new variable, and the value will be computed for the +posterior distribution of eight alternative specification. For example, +a value of mean(theta) will compute the posterior mean of +theta for each alternative specification.
    .preserve

    as in filter and slice

    		.model_data

    Stan model data, if not provided in the earlier call to +adjust_weights.
    +

    Value

    + +

    An adjustr_weighted object, with the new columns specified in +... added.