-
Notifications
You must be signed in to change notification settings - Fork 5
tmatta edited this page Oct 16, 2017
·
2 revisions
We examined laninet trait recovery from nine different populations under the following conditions: 1 (IRT models) x 1 (IRT R packages) x 3 (sample sizes) x 3 (test lengths) x 3 (test booklets). We used TAM (Robitzsch, Kiefer, & Wu, 2017) to estimate the population-specific distributions and compare the results to the generating values.
- One IRT model was included: Rasch model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value
- One IRT R packages was evaluated:
TAM(version 2.4-9) - Three sample sizes in each group were used: 500, 1000, and 5000
- Group 1 : θ1 ~ N(0, 1.00)
- Group 2 : θ2 ~ N(-2, 0.75)
- Group 3 : θ3 ~ N(-1.5, 1.50)
- Group 4 : θ4 ~ N(-1, 1)
- Group 5 : θ5 ~ N(-0.5, 0.75)
- Group 6 : θ6 ~ N(0.5, 1.50)
- Group 7 : θ7 ~ N(1, 1)
- Group 8 : θ8 ~ N(1.5, 0.75)
- Group 9 : θ9 ~ N(2, 1.50)
- Three test lengths were used: 40, 60, and 80 items
- Three test booklets were used: 5, 10, and 15 booklets
- One hundred replications were used for each condition for the calibration
- Person trait recovery:
- Across all conditions, the latent trait distributions were recovered well
| Country | True_mean | Est_mean | True_SD | Est_SD |
|---|---|---|---|---|
| 1 | 0.0 | 0.000 | 1.00 | 0.999 |
| 2 | -2.0 | -2.008 | 0.75 | 0.748 |
| 3 | -1.5 | -1.498 | 1.50 | 1.487 |
| 4 | -1.0 | -1.011 | 1.00 | 0.999 |
| 5 | -0.5 | -0.508 | 0.75 | 0.750 |
| 6 | 0.5 | 0.497 | 1.50 | 1.497 |
| 7 | 1.0 | 0.987 | 1.00 | 0.998 |
| 8 | 1.5 | 1.491 | 0.75 | 0.747 |
| 9 | 2.0 | 1.974 | 1.50 | 1.467 |
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #2.4-9
}# Define population distributions
true_mean <- c(0, -2.0, -1.5, -1.0, -0.5, 0.5, 1.0, 1.5, 2.0)
true_sd <- rep(c(1, .75, 1.5), 3)# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80) #number of items
K.cond <- c(5, 10, 15) #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results_mean <- NULL
results_sd <- NULL#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of items
# generate item parameters for a Rasch model
set.seed(2607) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_1pl = I,
b_bounds = c(-2, 2))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
set.seed(8088*k+1)
# generate thetas for nine groups
theta_1 <- data.frame(theta = rnorm(N, true_mean[1], true_sd[1]))
theta_2 <- data.frame(theta = rnorm(N, true_mean[2], true_sd[2]))
theta_3 <- data.frame(theta = rnorm(N, true_mean[3], true_sd[3]))
theta_4 <- data.frame(theta = rnorm(N, true_mean[4], true_sd[4]))
theta_5 <- data.frame(theta = rnorm(N, true_mean[5], true_sd[5]))
theta_6 <- data.frame(theta = rnorm(N, true_mean[6], true_sd[6]))
theta_7 <- data.frame(theta = rnorm(N, true_mean[7], true_sd[7]))
theta_8 <- data.frame(theta = rnorm(N, true_mean[8], true_sd[8]))
theta_9 <- data.frame(theta = rnorm(N, true_mean[9], true_sd[9]))
theta <- rbind(theta_1, theta_2, theta_3,
theta_4, theta_5, theta_6,
theta_7, theta_8, theta_9)
### -- Cognitive assessment generation
# assign items to blocks
blocks <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool,
item_block_matrix = NULL)
#assign blocks to booklets
books <- lsasim::booklet_design(item_block_assignment = blocks$block_assignment,
book_design = NULL)
#assign booklets to subjects
book_samp <- lsasim::booklet_sample(n_subj = N*length(true_mean),
book_item_design = books,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta$theta,
b_par = item_pool$b)
# assign country
cog$country <- rep(1:length(true_mean), each = N)
#------------------------------------------------------------------------------#
# Model estimation
#------------------------------------------------------------------------------#
# fit Rasch model using TAM package
# where the population model assumes separate normal distributions for the
# nine countries
mod <- NULL
mod <- TAM::tam.mml(cog[,1:I], group = cog$country)
mod_results_mean <- matrix( c(N, I, K, as.numeric(mod$beta), k), nrow=1)
colnames(mod_results_mean) <- c("N", "I", "K",
paste0("cnt", 1:9, "_est_mean"),
"rep")
# summarize results
mod_results_sd <- cbind(N, I, K,
sqrt(mean(mod$variance[cog$country == 1])),
sqrt(mean(mod$variance[cog$country == 2])),
sqrt(mean(mod$variance[cog$country == 3])),
sqrt(mean(mod$variance[cog$country == 4])),
sqrt(mean(mod$variance[cog$country == 5])),
sqrt(mean(mod$variance[cog$country == 6])),
sqrt(mean(mod$variance[cog$country == 7])),
sqrt(mean(mod$variance[cog$country == 8])),
sqrt(mean(mod$variance[cog$country == 9])),
r)
colnames(mod_results_sd) <- c("N", "I", "K", paste0("cnt", 1:9, "_est_sd"), "rep")
# combine results
results_mean <- rbind(results_mean, mod_results_mean)
results_sd <- rbind(results_sd, mod_results_sd)
}
}
}
}
Summary:
- Theta distributions for nine groups across all conditions
est_mean <- round(colMeans(results_mean[,c(4:12)]),3)
est_sd <- round(colMeans(results_sd[,c(4:12)]),3)
theta.dists <- data.frame(cbind(paste0("cnt", 1:9), true_mean, true_sd, est_mean, est_sd))
names(theta.dists) <- c("Country", "Gen_mean", "Gen_sd", "Est_mean", "Est_sd")
row.names(theta.dists) <- NULL
print(theta.dists)## Country Gen_mean Gen_sd Est_mean Est_sd
## 1 cnt1 0 1 0 0.999
## 2 cnt2 -2 0.75 -2.008 0.748
## 3 cnt3 -1.5 1.5 -1.498 1.487
## 4 cnt4 -1 1 -1.011 0.999
## 5 cnt5 -0.5 0.75 -0.508 0.75
## 6 cnt6 0.5 1.5 0.497 1.497
## 7 cnt7 1 1 0.987 0.998
## 8 cnt8 1.5 0.75 1.491 0.747
## 9 cnt9 2 1.5 1.974 1.467
- Estimated means for the nine countries under full conditions
thetas_mean <- aggregate(cbind(cnt1_est_mean, cnt2_est_mean, cnt3_est_mean,
cnt4_est_mean, cnt5_est_mean, cnt6_est_mean,
cnt7_est_mean, cnt8_est_mean, cnt9_est_mean) ~ N + I + K,
data=results_mean, mean, na.rm=TRUE)
names(thetas_mean) <- c("N", "I", "K", paste0("cnt", 1:9))print(round(thetas_mean, 3)) #estimated means## N I K cnt1 cnt2 cnt3 cnt4 cnt5 cnt6 cnt7 cnt8 cnt9
## 1 500 40 5 0 -2.013 -1.501 -1.019 -0.519 0.491 0.966 1.480 1.968
## 2 1000 40 5 0 -2.008 -1.501 -1.016 -0.508 0.493 0.992 1.488 1.979
## 3 5000 40 5 0 -2.004 -1.496 -1.001 -0.499 0.501 1.000 1.498 1.984
## 4 500 60 5 0 -2.014 -1.501 -1.011 -0.514 0.499 0.973 1.487 1.981
## 5 1000 60 5 0 -2.006 -1.496 -1.018 -0.505 0.495 0.995 1.489 1.980
## 6 5000 60 5 0 -2.003 -1.499 -1.001 -0.500 0.502 0.999 1.498 1.986
## 7 500 80 5 0 -2.005 -1.497 -1.009 -0.512 0.500 0.973 1.486 1.978
## 8 1000 80 5 0 -2.006 -1.503 -1.017 -0.508 0.493 0.993 1.490 1.978
## 9 5000 80 5 0 -2.004 -1.498 -1.001 -0.500 0.501 1.000 1.498 1.991
## 10 500 40 10 0 -2.016 -1.497 -1.014 -0.514 0.491 0.966 1.482 1.953
## 11 1000 40 10 0 -2.006 -1.496 -1.018 -0.505 0.493 0.988 1.492 1.970
## 12 5000 40 10 0 -2.008 -1.497 -1.005 -0.501 0.498 0.999 1.496 1.974
## 13 500 60 10 0 -2.015 -1.493 -1.013 -0.513 0.500 0.974 1.488 1.972
## 14 1000 60 10 0 -1.999 -1.499 -1.015 -0.502 0.502 0.995 1.495 1.974
## 15 5000 60 10 0 -2.004 -1.496 -0.999 -0.498 0.502 1.001 1.500 1.983
## 16 500 80 10 0 -2.006 -1.498 -1.012 -0.516 0.496 0.973 1.489 1.975
## 17 1000 80 10 0 -2.006 -1.500 -1.015 -0.503 0.493 0.994 1.491 1.979
## 18 5000 80 10 0 -2.005 -1.496 -1.001 -0.498 0.504 1.000 1.500 1.984
## 19 500 40 15 0 -2.025 -1.493 -1.015 -0.521 0.499 0.974 1.483 1.962
## 20 1000 40 15 0 -2.008 -1.496 -1.021 -0.507 0.490 0.986 1.484 1.961
## 21 5000 40 15 0 -2.005 -1.490 -1.002 -0.501 0.501 0.999 1.495 1.965
## 22 500 60 15 0 -2.017 -1.497 -1.019 -0.520 0.496 0.971 1.485 1.962
## 23 1000 60 15 0 -2.010 -1.501 -1.020 -0.511 0.492 0.989 1.490 1.971
## 24 5000 60 15 0 -2.005 -1.495 -1.005 -0.503 0.499 0.996 1.498 1.973
## 25 500 80 15 0 -2.015 -1.499 -1.016 -0.523 0.490 0.966 1.486 1.959
## 26 1000 80 15 0 -2.002 -1.498 -1.016 -0.506 0.496 0.993 1.489 1.975
## 27 5000 80 15 0 -2.003 -1.497 -1.001 -0.499 0.502 0.998 1.498 1.977
- Estimated standard deviations for the nine countries under full conditions
thetas_sd <- aggregate(cbind(cnt1_est_sd, cnt2_est_sd, cnt3_est_sd,
cnt4_est_sd, cnt5_est_sd, cnt6_est_sd,
cnt7_est_sd, cnt8_est_sd, cnt9_est_sd) ~ N + I + K,
data=results_sd, mean, na.rm=TRUE)
names(thetas_sd) <- c("N", "I", "K", paste0("cnt", 1:9))print(round(thetas_sd, 3)) #estimated sds## N I K cnt1 cnt2 cnt3 cnt4 cnt5 cnt6 cnt7 cnt8 cnt9
## 1 500 40 5 0.999 0.744 1.493 0.999 0.750 1.491 1.000 0.752 1.471
## 2 1000 40 5 0.994 0.752 1.489 0.995 0.750 1.499 0.999 0.751 1.475
## 3 5000 40 5 0.998 0.751 1.491 1.001 0.752 1.497 1.000 0.752 1.472
## 4 500 60 5 0.999 0.752 1.493 0.998 0.746 1.502 0.992 0.743 1.475
## 5 1000 60 5 0.999 0.752 1.489 1.001 0.749 1.498 0.999 0.750 1.477
## 6 5000 60 5 1.001 0.747 1.495 1.003 0.753 1.497 1.000 0.752 1.478
## 7 500 80 5 0.998 0.747 1.494 0.998 0.743 1.496 0.999 0.742 1.479
## 8 1000 80 5 0.997 0.749 1.495 0.998 0.750 1.502 0.997 0.753 1.483
## 9 5000 80 5 1.000 0.751 1.496 1.002 0.752 1.497 1.000 0.750 1.483
## 10 500 40 10 0.998 0.735 1.480 0.991 0.744 1.503 0.995 0.744 1.448
## 11 1000 40 10 1.001 0.750 1.483 0.998 0.750 1.499 0.997 0.747 1.460
## 12 5000 40 10 0.998 0.748 1.484 1.003 0.751 1.495 1.001 0.752 1.460
## 13 500 60 10 0.994 0.753 1.483 0.995 0.752 1.494 0.999 0.738 1.470
## 14 1000 60 10 0.999 0.751 1.488 0.998 0.742 1.501 0.994 0.751 1.471
## 15 5000 60 10 1.001 0.748 1.491 1.001 0.751 1.496 1.004 0.751 1.469
## 16 500 80 10 0.998 0.747 1.500 1.000 0.752 1.496 1.001 0.745 1.481
## 17 1000 80 10 0.998 0.748 1.492 0.998 0.753 1.498 0.998 0.749 1.473
## 18 5000 80 10 1.000 0.753 1.492 1.004 0.753 1.498 1.001 0.750 1.473
## 19 500 40 15 1.005 0.736 1.472 0.995 0.749 1.488 1.003 0.728 1.441
## 20 1000 40 15 0.997 0.755 1.473 1.002 0.750 1.493 0.995 0.739 1.456
## 21 5000 40 15 0.998 0.746 1.481 1.000 0.753 1.499 1.003 0.750 1.442
## 22 500 60 15 0.997 0.747 1.483 0.989 0.748 1.492 0.992 0.743 1.464
## 23 1000 60 15 1.001 0.745 1.484 0.994 0.748 1.502 0.995 0.750 1.466
## 24 5000 60 15 0.998 0.751 1.484 1.004 0.752 1.498 1.000 0.752 1.459
## 25 500 80 15 1.002 0.738 1.482 0.993 0.752 1.490 0.996 0.745 1.462
## 26 1000 80 15 0.996 0.746 1.484 0.997 0.749 1.497 0.995 0.750 1.465
## 27 5000 80 15 0.999 0.748 1.490 1.003 0.750 1.497 1.000 0.749 1.464