-
Notifications
You must be signed in to change notification settings - Fork 0
/
IRRsimulations.R
executable file
·70 lines (63 loc) · 2.37 KB
/
IRRsimulations.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
library(IRRsim)
# Directory to save the simulated data
out_dir <- 'data-releases'
# Set run_parallel = FALSE to ensure subsequent runs generate the same data.
run_parallel <- TRUE
set.seed(2112)
# nRaters = 2:12 # Number of raters
nRaters <- c(2, 4, 8, 16)
nLevels <- 2:5 # The number of scoring levels
IRRsimulations <- list()
for(i in nLevels) {
probability.weights <- list(
'Uniform' = rep(1/i, i),
'Lightly' = c(rep(.55 / (i - 1), i - 1), .55),
'Moderately' = c(rep(.4 / (i - 1), i - 1), .6),
'Highly' = c(rep(.2 / (i - 1), i - 1), .8)
)
for(j in seq_len(length(probability.weights))) {
print(paste0('Simulating ', i, ' levels with ',
names(probability.weights)[j], ' distribution'))
for(k in nRaters) {
for(k_per_event in seq(2, k)) {
tmp <- simulateIRR(nRaters = k,
nRatersPerEvent = k_per_event,
nLevels = i,
response.probs = probability.weights[[j]],
parallel = run_parallel,
showTextProgress = FALSE)
tmp2 <- unlist(sapply(tmp, FUN = function(x) { as.integer(x$data) }))
IRRsimulations[[length(IRRsimulations) + 1]] <- list(
raw = tmp,
nLevels = i,
nRaters = k,
nRatersPerEvent = k_per_event,
response.probs.name = names(probability.weights)[j],
response.probs = probability.weights[[j]],
response.dist = prop.table(table(tmp2))
)
}
}
}
}
save(IRRsimulations, file = paste0(out_dir, '/IRRsimulations.rda'))
# Create a data.frame combining the results of the simulations
load(paste0(out_dir, '/IRRsimulations.rda'))
IRRsimData <- data.frame()
for(i in seq_len(length(IRRsimulations))) {
tmp <- as.data.frame(IRRsimulations[[i]]$raw)
tmp <- tmp[,!names(tmp) %in% c('Cohen_Kappa')]
# tmp$ResponseDist <- IRRsimulations[[i]]$response.probs.name
IRRsimData <- rbind(IRRsimData, tmp)
}
IRRsimData$nLevels <- factor(IRRsimData$nLevels, ordered = TRUE)
IRRsimData$PercentRated <- IRRsimData$k_per_event / IRRsimData$k
IRRsimData$k <- factor(IRRsimData$k, ordered = TRUE)
IRRsimData$k_per_event <- factor(IRRsimData$k_per_event, ordered = TRUE)
IRRsimData$ResponseDist <- cut(IRRsimData$MaxResponseDiff,
breaks = c(-Inf, 0.1, 0.2, 0.3, Inf),
labels = c('Uniform', 'Lightly Skewed', 'Moderately Skewed', 'Highly Skewed'),
ordered = TRUE)
save(IRRsimData, file = paste0(out_dir, '/IRRsimData.rda'))
tools::resaveRdaFiles(out_dir)
rm(IRRsimulations)