-
Notifications
You must be signed in to change notification settings - Fork 5
Multiple booklets 2PL items
tmatta edited this page Oct 17, 2017
·
3 revisions
We examined item parameter recovery under the following conditions: 1 (IRT model) x 2 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 3 (test booklets)
- One IRT model was included: 2PL model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value - The bounds of the item discrimination parameter, a, are constrained to
a_bounds = (0.75, 1.25)where 0.75 is the lowest generating value and 1.25 is the highest generating value
- Two IRT R packages were evaluated:
TAM(version 2.4-9) andmirt(version 1.25) - Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- Three test booklets were used: 5, 10, and 15 booklets
- One hundred replications were used for each condition for the calibration.
- Summary of item parameter recovery:
-
TAMandmirtdemonstrated a similar level of accuracy - Sample sizes of 5000 consistently produced more accurate results
- For b- and a-parameters, recovery accuracy didn't perform well when small sample size, fewer items, and more number of booklets
- For b- and a-parameters, when sample size increased, recover accuracy improved further
- For b- and a-parameters, when number of items increased, recover accuracy improved further
- For b- and a-parameters, when number of booklets increased, recovery accuracy decreased
-
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- c(5, 10, 15) #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of items
# generate item parameters for a 2PL model
set.seed(4364) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_2pl = I,
thresholds = 1,
b_bounds = c(-2, 2),
a_bounds = c(0.75, 1.25))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
set.seed(12345 + r)
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to blocks
blocks <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool,
item_block_matrix = NULL)
#assign blocks to booklets
books <- lsasim::booklet_design(item_block_assignment
= blocks$block_assignment,
book_design = NULL)
#assign booklets to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = books,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta,
b_par = item_pool$b,
a_par = item_pool$a)
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit 2PL model using mirt package
mirt.mod <- NULL
mirt.mod <- mirt::mirt(resp, 1, itemtype = '2PL', verbose = F,
technical = list( NCYCLES = 500))
# fit 2PL model using TAM package
tam.mod <- NULL
tam.mod <- TAM::tam.mml.2pl(resp, control = list(maxiter = 200))
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# extract b, a in mirt package
mirt_b <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b"]
mirt_a <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"a"]
# convert TAM output into 2PL parametrization
tam_b <- (tam.mod$item$xsi.item) / (tam.mod$item$B.Cat1.Dim1)
tam_a <- (tam.mod$item$B.Cat1.Dim1)
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow=1))
colnames(itempars) <- c("N", "I", "K", "rep")
# calculate corr, bias, RMSE for item parameters in mirt pacakge
itempars$corr_mirt_b <- cor( item_pool$b, mirt_b)
itempars$bias_mirt_b <- mean( mirt_b - item_pool$b )
itempars$RMSE_mirt_b <- sqrt(mean( ( mirt_b - item_pool$b)^2 ))
itempars$corr_mirt_a <- cor( item_pool$a, mirt_a)
itempars$bias_mirt_a <- mean( mirt_a - item_pool$a )
itempars$RMSE_mirt_a <- sqrt(mean( ( mirt_a - item_pool$a)^2 ))
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b <- cor( item_pool$b, tam_b)
itempars$bias_tam_b <- mean( tam_b - item_pool$b )
itempars$RMSE_tam_b <- sqrt(mean( ( tam_b - item_pool$b)^2 ))
itempars$corr_tam_a <- cor( item_pool$a, tam_a)
itempars$bias_tam_a <- mean( tam_a - item_pool$a )
itempars$RMSE_tam_a <- sqrt(mean( ( tam_a - item_pool$a)^2 ))
# combine results
results <- rbind(results, itempars)
}
}
}
}
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
mirt_recovery <- aggregate(cbind(corr_mirt_b, bias_mirt_b, RMSE_mirt_b,
corr_mirt_a, bias_mirt_a, RMSE_mirt_a) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(mirt_recovery) <- c("Sample Size", "Test Length", "# of Booklets",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a")
round(mirt_recovery, 3)## Sample Size Test Length # of Booklets corr_b bias_b RMSE_b corr_a bias_a RMSE_a
## 1 500 40 5 0.960 0.006 0.403 0.463 0.041 0.301
## 2 1000 40 5 0.985 -0.005 0.233 0.601 0.019 0.196
## 3 5000 40 5 0.997 0.004 0.098 0.866 0.003 0.084
## 4 500 60 5 0.967 -0.002 0.381 0.479 0.034 0.280
## 5 1000 60 5 0.986 0.002 0.239 0.635 0.021 0.186
## 6 5000 60 5 0.997 0.002 0.100 0.870 0.003 0.081
## 7 500 80 5 0.963 -0.013 0.367 0.516 0.032 0.255
## 8 1000 80 5 0.985 -0.003 0.221 0.663 0.019 0.173
## 9 5000 80 5 0.997 0.001 0.092 0.896 0.001 0.075
## 10 500 100 5 0.965 -0.005 0.352 0.536 0.029 0.246
## 11 1000 100 5 0.985 -0.002 0.223 0.681 0.017 0.169
## 12 5000 100 5 0.997 0.001 0.094 0.900 0.002 0.074
## 13 500 40 10 0.676 -0.093 3.056 0.259 0.178 0.862
## 14 1000 40 10 0.919 0.060 0.952 0.366 0.058 0.384
## 15 5000 40 10 0.992 0.000 0.161 0.713 0.011 0.143
## 16 500 60 10 0.726 0.896 8.756 0.289 0.114 0.634
## 17 1000 60 10 0.931 0.017 0.614 0.413 0.045 0.340
## 18 5000 60 10 0.993 0.004 0.161 0.739 0.008 0.132
## 19 500 80 10 0.765 0.015 3.584 0.328 0.087 0.482
## 20 1000 80 10 0.954 0.003 0.419 0.483 0.037 0.289
## 21 5000 80 10 0.993 0.003 0.147 0.784 0.006 0.120
## 22 500 100 10 0.834 -0.025 1.118 0.361 0.076 0.446
## 23 1000 100 10 0.959 -0.005 0.383 0.510 0.037 0.275
## 24 5000 100 10 0.994 0.001 0.141 0.803 0.006 0.114
## 25 500 40 15 0.373 -0.051 8.794 0.166 0.861 2.917
## 26 1000 40 15 0.737 -0.127 3.564 0.256 0.166 0.804
## 27 5000 40 15 0.982 0.004 0.250 0.574 0.022 0.222
## 28 500 60 15 0.440 -0.345 12.158 0.162 0.488 1.975
## 29 1000 60 15 0.777 0.152 3.629 0.283 0.136 0.680
## 30 5000 60 15 0.986 0.002 0.233 0.603 0.014 0.194
## 31 500 80 15 0.463 0.029 6.604 0.211 0.239 1.178
## 32 1000 80 15 0.816 -0.160 3.936 0.349 0.077 0.472
## 33 5000 80 15 0.988 0.003 0.196 0.686 0.010 0.162
## 34 500 100 15 0.487 0.465 20.956 0.235 0.191 0.945
## 35 1000 100 15 0.849 0.021 1.078 0.376 0.064 0.402
## 36 5000 100 15 0.988 0.002 0.192 0.709 0.009 0.154
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
tam_recovery <- aggregate(cbind(corr_tam_b, bias_tam_b, RMSE_tam_b,
corr_tam_a, bias_tam_a, RMSE_tam_a) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(tam_recovery) <- c("Sample Size", "Test Length", "# of Booklets",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a")
round(tam_recovery, 3)## Sample Size Test Length # of Booklets corr_b bias_b RMSE_b corr_a bias_a RMSE_a
## 1 500 40 5 0.960 0.006 0.403 0.463 0.041 0.301
## 2 1000 40 5 0.985 -0.005 0.233 0.601 0.020 0.196
## 3 5000 40 5 0.997 0.004 0.098 0.866 0.003 0.084
## 4 500 60 5 0.967 -0.002 0.380 0.479 0.034 0.280
## 5 1000 60 5 0.986 0.003 0.239 0.635 0.021 0.186
## 6 5000 60 5 0.997 0.003 0.100 0.870 0.003 0.081
## 7 500 80 5 0.963 -0.013 0.367 0.516 0.032 0.255
## 8 1000 80 5 0.985 -0.003 0.221 0.663 0.019 0.173
## 9 5000 80 5 0.997 0.001 0.092 0.896 0.001 0.075
## 10 500 100 5 0.965 -0.005 0.352 0.536 0.029 0.246
## 11 1000 100 5 0.985 -0.002 0.223 0.681 0.017 0.169
## 12 5000 100 5 0.997 0.002 0.094 0.900 0.002 0.074
## 13 500 40 10 0.680 1.602 13.053 0.263 0.179 0.881
## 14 1000 40 10 0.919 0.060 0.952 0.366 0.058 0.383
## 15 5000 40 10 0.992 0.000 0.161 0.713 0.011 0.143
## 16 500 60 10 0.729 -1.961 18.829 0.289 0.117 0.664
## 17 1000 60 10 0.931 0.017 0.614 0.413 0.045 0.340
## 18 5000 60 10 0.993 0.004 0.161 0.739 0.008 0.132
## 19 500 80 10 0.765 0.018 3.674 0.328 0.087 0.481
## 20 1000 80 10 0.954 0.003 0.419 0.483 0.037 0.289
## 21 5000 80 10 0.993 0.003 0.147 0.784 0.006 0.120
## 22 500 100 10 0.834 -0.025 1.121 0.361 0.076 0.447
## 23 1000 100 10 0.959 -0.005 0.383 0.510 0.037 0.275
## 24 5000 100 10 0.994 0.001 0.141 0.803 0.006 0.114
## 25 500 40 15 0.381 0.451 10.628 0.166 0.771 2.917
## 26 1000 40 15 0.736 -0.018 2.903 0.258 0.159 0.779
## 27 5000 40 15 0.982 0.004 0.250 0.573 0.022 0.222
## 28 500 60 15 0.442 -0.882 18.078 0.156 0.432 1.900
## 29 1000 60 15 0.777 0.094 3.190 0.286 0.130 0.641
## 30 5000 60 15 0.986 0.002 0.233 0.603 0.014 0.194
## 31 500 80 15 0.462 -0.338 10.116 0.208 0.253 1.317
## 32 1000 80 15 0.816 -0.156 3.893 0.350 0.078 0.483
## 33 5000 80 15 0.988 0.003 0.196 0.686 0.010 0.162
## 34 500 100 15 0.486 0.926 26.723 0.236 0.192 0.978
## 35 1000 100 15 0.849 0.021 1.078 0.376 0.064 0.402
## 36 5000 100 15 0.988 0.002 0.192 0.709 0.009 0.154