ArchiveTuning and extract_inner_tuning_* functions

Adds function to extract archives of inner resampling loop.
Adds runtime to archive.
New ArchiveTuning documentation with "data structure" and "analysis" section.
as.data.table.ArchiveTuning() can unnest columns, exlcude columns and score the hyperparameter configurations on additional performance measures.
Calling as.data.table.ArchiveTuning() and extract_inner_tuning_archives joins the resample results to the table. Removes ArchiveTuning$extended_archive() method.

resample

task = tsk("iris")
learner = lrn("classif.rpart")
learner$param_set$values$cp = to_tune(0.001, 0.1)

at = AutoTuner$new(
  learner = learner,
  resampling = rsmp("holdout"),
  measure = msr("classif.ce"),
  terminator = trm("evals", n_evals = 5),
  tuner = tnr("grid_search"),
  store_tuning_instance = TRUE)

resampling_outer = rsmp("cv", folds = 2)
rr = resample(task, at, resampling_outer, store_models = TRUE)

extract_inner_tuning_archives(rr)

#>     iteration    cp classif.ce runtime           timestamp batch_nr x_domain_cp      resample_result task_id          learner_id resampling_id
#>  1:         1 0.045       0.12   0.009 2021-05-28 11:52:36        1       0.045 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  2:         1 0.023       0.12   0.009 2021-05-28 11:52:36        4       0.023 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  3:         1 0.078       0.12   0.009 2021-05-28 11:52:36        2       0.078 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  4:         1 0.012       0.12   0.009 2021-05-28 11:52:36        3       0.012 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  5:         1 0.001       0.12   0.009 2021-05-28 11:52:36        5       0.001 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  6:         2 0.012       0.04   0.008 2021-05-28 11:52:35        4       0.012 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  7:         2 0.023       0.04   0.009 2021-05-28 11:52:35        1       0.023 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  8:         2 0.034       0.04   0.009 2021-05-28 11:52:35        5       0.034 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#>  9:         2 0.001       0.04   0.008 2021-05-28 11:52:35        2       0.001 <ResampleResult[21]>    iris classif.rpart.tuned            cv
#> 10:         2 0.045       0.04   0.010 2021-05-28 11:52:35        3       0.045 <ResampleResult[21]>    iris classif.rpart.tuned            cv

benchmark


at_1 = AutoTuner$new(
  learner = lrn("classif.rpart", cp = to_tune(0.001, 0.1)),
  resampling = rsmp("holdout"),
  measure = msr("classif.ce"),
  terminator = trm("evals", n_evals = 5),
  tuner = tnr("grid_search"))

at_2 = AutoTuner$new(
  learner = lrn("classif.debug", x = to_tune()),
  resampling = rsmp("holdout"),
  measure = msr("classif.ce"),
  terminator = trm("evals", n_evals = 5),
  tuner = tnr("grid_search"))

grid = benchmark_grid(tsk("iris"), list(at_1, at_2), rsmp("cv", folds = 2))
bmr = benchmark(grid, store_models = TRUE)

extract_inner_tuning_archives(bmr)

#>     experiment iteration    cp         x classif.ce runtime           timestamp batch_nr x_domain_cp      resample_result task_id          learner_id resampling_id x_domain_x
#>  1:          1         1 0.023        NA       0.04   0.009 2021-05-28 11:53:21        2       0.023 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  2:          1         1 0.078        NA       0.04   0.009 2021-05-28 11:53:22        4       0.078 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  3:          1         1 0.089        NA       0.04   0.009 2021-05-28 11:53:22        3       0.089 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  4:          1         1 0.034        NA       0.04   0.010 2021-05-28 11:53:22        5       0.034 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  5:          1         1 0.045        NA       0.04   0.008 2021-05-28 11:53:21        1       0.045 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  6:          1         2 0.001        NA       0.08   0.008 2021-05-28 11:53:21        4       0.001 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  7:          1         2 0.067        NA       0.08   0.009 2021-05-28 11:53:21        2       0.067 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  8:          1         2 0.023        NA       0.08   0.009 2021-05-28 11:53:21        3       0.023 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#>  9:          1         2 0.100        NA       0.08   0.009 2021-05-28 11:53:21        5       0.100 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#> 10:          1         2 0.045        NA       0.08   0.009 2021-05-28 11:53:20        1       0.045 <ResampleResult[21]>    iris classif.rpart.tuned            cv         NA
#> 11:          2         1    NA 1.0000000       0.52   0.004 2021-05-28 11:53:19        2          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  1.0000000
#> 12:          2         1    NA 0.3333333       0.80   0.004 2021-05-28 11:53:19        5          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.3333333
#> 13:          2         1    NA 0.2222222       0.68   0.003 2021-05-28 11:53:19        4          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.2222222
#> 14:          2         1    NA 0.1111111       0.68   0.004 2021-05-28 11:53:19        3          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.1111111
#> 15:          2         1    NA 0.7777778       0.80   0.005 2021-05-28 11:53:19        1          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.7777778
#> 16:          2         2    NA 0.0000000       0.68   0.004 2021-05-28 11:53:20        4          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.0000000
#> 17:          2         2    NA 0.4444444       0.68   0.004 2021-05-28 11:53:20        5          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.4444444
#> 18:          2         2    NA 0.6666667       0.68   0.004 2021-05-28 11:53:20        3          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.6666667
#> 19:          2         2    NA 0.8888889       0.68   0.003 2021-05-28 11:53:20        1          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.8888889
#> 20:          2         2    NA 0.5555556       0.68   0.003 2021-05-28 11:53:20        2          NA <ResampleResult[21]>    iris classif.debug.tuned            cv  0.5555556

mlr-org / mlr3tuning

ArchiveTuning and extract_inner_tuning_* functions #301