Custom rolling window nested resampling with mlr3 - r

Question

Since last version, mlr3tuning package supports (custom) instantiated resampling in AutoTuner class: https://github.com/mlr-org/mlr3tuning/releases/tag/v0.17.2
I have tried to construct rolling window CV's with custom resmapling as in the following link https://towardsdatascience.com/time-series-nested-cross-validation-76adba623eb9 (figure 1).
I want to tune hyperparameters on (let's say )row ids 1:1000 (for example 1:800 train and 801:100 test). and then I would like to evaluate the model on test set, say 1001:1100.
Her is my try:
library(mlr3)
library(mlr3tuning)
library(mlr3pipelines)
library(mlr3learners)
# task
task = tsk("iris")
task_ = task$clone()
data_ = task_$data()
data_ = cbind(data_, monthid = c(rep(1, 30), rep(2, 30), rep(3, 30), rep(4, 30), rep(5, 30)))
task = as_task_classif(data_, target = "Species")
# inner custom rolling window resampling
custom = rsmp("custom")
task_ <- task$clone()
task_$set_col_roles("monthid", "group")
groups = task_$groups
rm(task_)
groups_v <- groups[, unique(group)]
train_length <- 2
test_length <- 1
train_groups <- lapply(0:(length(groups_v)-(train_length+1)), function(x) x + (1:train_length))
test_groups <- lapply(train_groups, function(x) tail(x, 1) + test_length)
train_sets <- lapply(train_groups, function(x) groups[group %in% groups_v[x], row_id])
test_sets <- lapply(test_groups, function(x) groups[group %in% groups_v[x], row_id])
custom$instantiate(task, train_sets, test_sets)
# outer custom rolling window resampling
customo = rsmp("custom")
task_ <- task$clone()
task_$set_col_roles("monthid", "group")
groups = task_$groups
rm(task_)
groups_v <- groups[, unique(group)]
train_length_out <- train_length + test_length
test_length_out <- 1
train_groups_out <- lapply(0:(length(groups_v)-(train_length_out+1)), function(x) x + (1:train_length_out))
test_groups_out <- lapply(train_groups_out, function(x) tail(x, 1) + test_length_out)
train_sets_out <- lapply(train_groups_out, function(x) groups[group %in% groups_v[x], row_id])
test_sets_out <- lapply(test_groups_out, function(x) groups[group %in% groups_v[x], row_id])
customo$instantiate(task, train_sets_out, test_sets_out)
# inspect custom cv's
custom$train_set(1)
custom$test_set(1)
(max(custom$train_set(1)) + 1) == head(custom$test_set(1), 1) # test set starts after train set
customo$train_set(1)
customo$test_set(1)
(max(customo$train_set(1)) + 1) == head(customo$test_set(1), 1) # test set starts after train set
all(c(custom$train_set(1), custom$test_set(1)) %in% customo$train_set(1)) # first outer set contains all inner sets
length(intersect(customo$test_set(1), c(custom$train_set(1), custom$test_set(1)))) == 0
# costruct graph
graph = po("removeconstants", id = "removeconstants_1", ratio = 0) %>>%
po("branch", options = c("nop_prep", "yeojohnson", "pca", "ica"), id = "prep_branch") %>>%
gunion(list(po("nop", id = "nop_prep"), po("yeojohnson"), po("pca", scale. = TRUE), po("ica"))) %>>%
po("unbranch", id = "prep_unbranch") %>>%
po("learner", learner = lrn("classif.rpart"))
plot(graph)
graph_learner = as_learner(graph)
as.data.table(graph_learner$param_set)[1:70, .(id, class, lower, upper)]
search_space = ps(
prep_branch.selection = p_fct(levels = c("nop_prep", "yeojohnson", "pca", "ica")),
pca.rank. = p_int(2, 6, depends = prep_branch.selection == "pca"),
ica.n.comp = p_int(2, 6, depends = prep_branch.selection == "ica"),
yeojohnson.standardize = p_lgl(depends = prep_branch.selection == "yeojohnson")
)
at = auto_tuner(
method = "random_search",
learner = graph_learner,
resampling = custom,
measure = msr("classif.acc"),
search_space = search_space
)
# resmpling
rr = resample(task, at, customo, store_models = TRUE)
I get an error:
INFO [09:46:49.340] [mlr3] Applying learner 'removeconstants_1.prep_branch.nop_prep.yeojohnson.pca.ica.prep_unbranch.classif.rpart.tuned' on task 'data_' (iter 1/2)
INFO [09:46:49.362] [mlr3] Applying learner 'removeconstants_1.prep_branch.nop_prep.yeojohnson.pca.ica.prep_unbranch.classif.rpart.tuned' on task 'data_' (iter 2/2)
Error: Train set 3 of inner resampling 'custom' contains row ids not present in task 'data_': {91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120}

Answer 1

The first outer training set contains these ids:
train_sets_out
#> [[1]]
#> [1] 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
This means row 1 to 90 are available in the inner resampling.
The third training set of the inner resampling needs rows 61 to 120 but rows 91 to 120 are not available.
train_sets
#> [[3]]
#> [1] 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
So that can't work. Check out figure 4.7 in the book to see how the outer and inner resampling work together.