Closed fweber144 closed 11 months ago
Illustration:
devtools::load_all()
dat_gauss <- data.frame(y = df_gaussian$y, df_gaussian$x)
rfit <- rstanarm::stan_glm(
y ~ X1 + X2 + X3 + X4 + X5, family = gaussian(), data = dat_gauss,
QR = TRUE, chains = 1, iter = 1000, refresh = 0, seed = 9876
)
# varsel ------------------------------------------------------------------
vs_search <- varsel(rfit, method = "L1", nterms_max = 3, refit_prj = FALSE,
seed = 5555)
vs_eval <- varsel(vs_search,
nclusters_pred = 10,
seed = 6666)
# cv_varsel ---------------------------------------------------------------
## `validate_search = FALSE` ----------------------------------------------
### PSIS-LOO CV -----------------------------------------------------------
cvvs_search_valF_loo <- cv_varsel(rfit, validate_search = FALSE, method = "L1",
nterms_max = 3, refit_prj = FALSE, seed = 5555)
cvvs_eval_valF_loo <- cv_varsel(cvvs_search_valF_loo,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valF_loo_from_vs <- cv_varsel(vs_search,
validate_search = FALSE,
nclusters_pred = 10,
seed = 6666)
### K-fold CV -------------------------------------------------------------
cvvs_search_valF <- cv_varsel(rfit, validate_search = FALSE, method = "L1",
cv_method = "kfold", K = 2,
nterms_max = 3, nclusters_pred = 2, seed = 5555)
cvvs_eval_valF <- cv_varsel(cvvs_search_valF,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valF_from_vs <- cv_varsel(vs_search,
validate_search = FALSE,
cv_method = "kfold", K = 2,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valF_from_loo <- cv_varsel(cvvs_search_valF_loo,
cv_method = "kfold", K = 2,
nclusters_pred = 10,
seed = 6666)
### PSIS-LOO CV with old K-fold CV results --------------------------------
cvvs_eval_valF_loo_from_valF <- cv_varsel(cvvs_search_valF,
cv_method = "LOO",
nclusters_pred = 10,
seed = 6666)
## `validate_search = TRUE` -----------------------------------------------
### PSIS-LOO CV -----------------------------------------------------------
cvvs_search_valT_loo <- cv_varsel(rfit, method = "L1", nterms_max = 3,
refit_prj = FALSE, seed = 5555)
cvvs_eval_valT_loo <- cv_varsel(cvvs_search_valT_loo,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_loo_from_vs <- cv_varsel(vs_search,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_loo_from_valF_loo <- cv_varsel(cvvs_search_valF_loo,
validate_search = TRUE,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_loo_from_valF <- cv_varsel(cvvs_search_valF,
cv_method = "LOO",
validate_search = TRUE,
nclusters_pred = 10,
seed = 6666)
### K-fold CV -------------------------------------------------------------
cvvs_search_valT <- cv_varsel(rfit, method = "L1", cv_method = "kfold", K = 2,
nterms_max = 3, refit_prj = FALSE, seed = 5555)
cvvs_eval_valT <- cv_varsel(cvvs_search_valT,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_from_vs <- cv_varsel(vs_search,
cv_method = "kfold", K = 2,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_from_valF_loo <- cv_varsel(cvvs_search_valF_loo,
cv_method = "kfold", K = 2,
validate_search = TRUE,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_from_valF <- cv_varsel(cvvs_search_valF,
validate_search = TRUE,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valT_from_valT_loo <- cv_varsel(cvvs_search_valT_loo,
cv_method = "kfold", K = 2,
nclusters_pred = 10,
seed = 6666)
### PSIS-LOO CV with old K-fold CV results --------------------------------
cvvs_eval_valT_loo_from_valT <- cv_varsel(cvvs_search_valT,
cv_method = "LOO",
nclusters_pred = 10,
seed = 6666)
## `validate_search = FALSE` with old `validate_search = TRUE` resu -------
## lts
cvvs_eval_valF_loo_from_valT_loo <- cv_varsel(cvvs_search_valT_loo,
validate_search = FALSE,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valF_from_valT_loo <- cv_varsel(cvvs_search_valT_loo,
cv_method = "kfold", K = 2,
validate_search = FALSE,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valF_loo_from_valT <- cv_varsel(cvvs_search_valT,
cv_method = "LOO",
validate_search = FALSE,
nclusters_pred = 10,
seed = 6666)
cvvs_eval_valF_from_valT <- cv_varsel(cvvs_search_valT,
validate_search = FALSE,
nclusters_pred = 10,
seed = 6666)
# varsel() with old cv_varsel() results -----------------------------------
vs_eval_from_valF_loo <- varsel(cvvs_search_valF_loo,
nclusters_pred = 10,
seed = 6666)
vs_eval_from_valF <- varsel(cvvs_search_valF,
nclusters_pred = 10,
seed = 6666)
vs_eval_from_valT_loo <- varsel(cvvs_search_valT_loo,
nclusters_pred = 10,
seed = 6666)
vs_eval_from_valT <- varsel(cvvs_search_valT,
nclusters_pred = 10,
seed = 6666)
This amends #461 and #463 by allowing users to switch the CV method in a
cv_varsel.vsel()
call withvalidate_search = TRUE
(which could also mean to usevarsel()
output in such acv_varsel.vsel()
call). Note that this implies that the fold-wise searches will have to be re-run. The relevant commit is c6de8e459d3da9343e119130f6e17de8f4162c65. Following it, there are also some refactoring commits.