Merge pull request #27 from nhs-bnssg-analytics/26-optimiser-recognis…

…ing-waitlist-size

26 optimiser recognising waitlist size

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: NHSRtt
 Type: Package
 Title: Scenario plan for future elective scenarios using public NHS data
-Version: 0.1.4.9000
+Version: 0.1.5.9000
 Authors@R: c(
     person("Sebastian", "Fox", , "sebastian.fox3@nhs.net", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0002-2832-773X")),

NAMESPACE

@@ -8,6 +8,7 @@ export(get_rtt_data)
 export(optimise_capacity)
 importFrom(dplyr,across)
 importFrom(dplyr,any_of)
+importFrom(dplyr,arrange)
 importFrom(dplyr,between)
 importFrom(dplyr,bind_rows)
 importFrom(dplyr,case_when)

NEWS.md

@@ -1,5 +1,9 @@
-# NHSRtt 0.1.4.9000
 
+# NHSRtt 0.1.5.9000
+
+# NHSRtt 0.1.5
+
+* includes non-admitted data in get_rtt_data which was accidentally being excluded
 
 # NHSRtt 0.1.4
 

R/data.R

@@ -101,7 +101,7 @@ get_rtt_data <- function(type, url = "https://www.england.nhs.uk/statistics/stat
     type_filter <- "New Periods"
   }
 
-  xl_files <- xl_files[grepl(paste0("^", type_filter), names(xl_files))]
+  xl_files <- xl_files[grepl(type_filter, names(xl_files))]
 
   # filter for dates
   dts <- as.Date(paste0("01", substr_right(names(xl_files), 5)), "%d%b%y")

R/modelling.R

@@ -229,7 +229,7 @@ calibrate_capacity_renege_params <- function(referrals, incompletes, completes,
 #'
 #' @importFrom rlang .data
 #' @importFrom dplyr distinct bind_rows left_join tibble join_by mutate
-#'   case_when summarise select
+#'   case_when summarise select arrange
 #' @importFrom tidyr replace_na
 #'
 #' @return a tibble with fields for period_id, months_waited_id,
@@ -377,6 +377,7 @@ apply_params_to_projections <- function(capacity_projections, referrals_projecti
           node_inflow = referrals_projections[period]
         )
       ) |>
+      dplyr::arrange(months_waited_id) |>
       # add in the projections for  capacity
       mutate(
         capacity = capacity_projections[period]

R/optimiser.R

@@ -22,7 +22,8 @@
 #'
 #' @returns a capacity multiplier representing the annual change in capacity
 #'   (from the input t_1_capacity to a capacity at t = 13) to achieve the
-#'   desired target within the target tolerance
+#'   desired target within the target tolerance. The name of the returned object
+#'   provides an indication of whether the optimiser converged
 #' @export
 #'
 optimise_capacity <- function(t_1_capacity, referrals_projections,
@@ -54,7 +55,11 @@ optimise_capacity <- function(t_1_capacity, referrals_projections,
   # check values of capacity_param
   if (all(renege_capacity_params[["capacity_param"]] == 0)) {
     warning("Unable to optimise as no treatments in the calibration period")
-    return(NA)
+    change_proportion <- setNames(
+      NA,
+      nm = "no_calibration_treatments"
+    )
+    return(change_proportion)
   }
 
   # check whether target_bin is less than the greatest number of months waited
@@ -90,6 +95,15 @@ optimise_capacity <- function(t_1_capacity, referrals_projections,
   if (length(tolerance) > 1)
     stop("tolerance must be length 1")
 
+  # if no t_1 capacity then multiplier will not work
+  if (t_1_capacity == 0) {
+    change_proportion <- setNames(
+      NA,
+      nm = "no_starting_capacity"
+    )
+    return(change_proportion)
+  }
+
   # target calculation
   current_val <- incomplete_pathways |>
     mutate(
@@ -142,6 +156,10 @@ optimise_capacity <- function(t_1_capacity, referrals_projections,
   above_target <- FALSE
   below_target <- FALSE
 
+  # for gradients that treat the whole waiting list, we want to find the minimum
+  # gradient that achieves that, so we set a starting value of NULL
+  min_change_proportion <- NULL
+
   while (converged == FALSE) {
     # build linear model for monthly capacity
     lm_fit <- stats::lm(
@@ -183,7 +201,9 @@ optimise_capacity <- function(t_1_capacity, referrals_projections,
       )
 
     if (sum(proportion_at_highest_bin[["incompletes"]]) == 0) {
+      # this part of statement means the total on the waiting list is zero
       proportion_at_highest_bin <- 0
+      min_change_proportion <- min(min_change_proportion, change_proportion)
     } else {
       proportion_at_highest_bin <- proportion_at_highest_bin |>
         mutate(
@@ -201,20 +221,43 @@ optimise_capacity <- function(t_1_capacity, referrals_projections,
 
     if (abs(compare_with_target) < tolerance) {
       converged <- TRUE
+      change_proportion <- setNames(
+        change_proportion,
+        nm = "converged"
+      )
     } else {
       iteration <- iteration + 1
 
-      if (isTRUE(last_iteration_proportion == proportion_at_highest_bin)) {
+      if (isTRUE(last_iteration_proportion == proportion_at_highest_bin) &
+          !(proportion_at_highest_bin %in% c(0, 1))) {
+
         warning("parameter distribution means optimiser cannot meet target")
         converged <- TRUE
         change_proportion <- Inf
+        change_proportion <- setNames(
+          change_proportion,
+          nm = "not_converged_check"
+        )
       }
       last_iteration_proportion <- proportion_at_highest_bin
 
       if (iteration > max_iterations) {
         warning("optimiser failed to converge before maximum iteration reached")
         converged <- TRUE # set to true so while loop is exited
-        change_proportion <- NaN
+
+        if (is.null(min_change_proportion)) {
+          change_proportion <- NaN
+          change_proportion <- setNames(
+            change_proportion,
+            nm = "not_converged"
+          )
+        } else {
+          change_proportion <- min(change_proportion)
+          change_proportion <- setNames(
+            change_proportion,
+            nm = "waitlist_cleared"
+          )
+        }
       }
 
       if (compare_with_target > 0) {

R/utils.R

@@ -181,7 +181,7 @@ calculate_timestep_transitions <- function(referrals, incompletes, completes, ma
 
   referrals <- referrals |>
     dplyr::filter(
-      .data$period_id > 0 # not need for input at timestep t-1
+      .data$period_id > 0 # no need for input at timestep t-1
     ) |>
     mutate(
       months_waited_id = 0

tests/testthat/test-modelling.R

@@ -404,35 +404,50 @@ test_that("apply_params_to_projections functionality", {
   )
 
   expected <- dplyr::tibble(
-    period_id = c(1L,1L,1L,1L,
-                  1L,2L,2L,2L,2L,2L,3L,3L,3L,3L,3L,4L,
-                  4L,4L,4L,4L),
-    months_waited_id = c(1L,2L,3L,4L,
-                         0L,2L,3L,4L,1L,0L,3L,4L,2L,1L,0L,4L,
-                         3L,2L,1L,0L),
-    calculated_treatments = c(27.63423362,
-                              42.12923285,81.22443171,155.9708739,20.04122796,
-                              31.91646707,49.0484251,279.3180835,
-                              13.59862424,5.118400036,37.56314012,390.8925394,
-                              15.87698091,3.510835114,1.156504437,382.1254298,
-                              13.99962849,3.071035894,0.594326398,0.2095794),
-    reneges = c(-237.5428439,
-                -434.4623289,-839.2180671,-1615.793391,
-                154.3246808,-1370.490791,-2110.114129,-12048.53131,
-                -486.7234099,164.111124,-7086.420979,
-                -73939.51857,-2989.602255,-551.0384133,162.6055174,
-                -429331.8355,-15687.34635,-3434.775038,-554.0696711,
-                175.0267722),
-    incompletes = c(324.9086103,
-                    495.333096,954.9936354,1833.822517,235.6340912,
-                    1663.482934,2556.3988,14558.02938,708.7588769,
-                    266.7704759,8712.340773,90663.0542,
-                    3682.484151,814.2980541,268.2379782,528325.105,
-                    19355.83087,4246.002057,821.7133229,289.7636484),
-    input_treatments = c(327L,327L,
-                         327L,327L,327L,379L,379L,379L,379L,379L,449L,
-                         449L,449L,449L,449L,400L,400L,400L,400L,
-                         400L)
+    period_id = c(
+      1L,1L,1L,1L,1L,2L,2L,2L,
+      2L,2L,3L,3L,3L,3L,3L,4L,4L,
+      4L,4L,4L
+    ),
+    months_waited_id = c(
+      0L,1L,2L,3L,4L,0L,1L,2L,
+      3L,4L,0L,1L,2L,3L,4L,0L,1L,
+      2L,3L,4L
+    ),
+    calculated_treatments = c(
+      20.04122796,27.63423362,
+      42.12923285,81.22443171,155.9708739,
+      5.118400036,13.59862424,31.91646707,
+      49.0484251,279.3180835,1.156504437,
+      3.510835114,15.87698091,37.56314012,
+      390.8925394,0.2095794,0.594326398,
+      3.071035894,13.99962849,382.1254298
+    ),
+    reneges = c(
+      154.3246808,-237.5428439,
+      -434.4623289,-839.2180671,-1615.793391,
+      164.111124,-486.7234099,-1370.490791,
+      -2110.114129,-12048.53131,
+      162.6055174,-551.0384133,-2989.602255,
+      -7086.420979,-73939.51857,175.0267722,
+      -554.0696711,-3434.775038,-15687.34635,
+      -429331.8355
+    ),
+    incompletes = c(
+      235.6340912,324.9086103,
+      495.333096,954.9936354,1833.822517,
+      266.7704759,708.7588769,1663.482934,
+      2556.3988,14558.02938,268.2379782,
+      814.2980541,3682.484151,8712.340773,
+      90663.0542,289.7636484,821.7133229,
+      4246.002057,19355.83087,528325.105
+    ),
+    input_treatments = c(
+      327L,327L,327L,327L,327L,
+      379L,379L,379L,379L,379L,449L,
+      449L,449L,449L,449L,400L,400L,400L,
+      400L,400L
+    )
   )
 
   expect_equal(

tests/testthat/test-optimiser.R

@@ -319,7 +319,7 @@ test_that("optimise_capacity functionality", {
       target = "~-2%",
       tolerance = 0.005
     ),
-    0.71875,
+    c(converged = 0.71875),
     info = "optimise_capacity consistently produces an answer"
   )
 })