Scale columns of a data.table to unit interval - r

Question

I have a data.table with a mixture of numeric and factor data, such as:
R> dat
x z y w
1: 3.307590 -1.66951137 b a
2: 1.809447 4.10331322 b b
3: 3.314621 3.69436879 a a
4: 1.896529 -0.08143017 c b
5: 3.317341 1.01839533 c a
6: 1.806456 -2.09547272 a b
...
I need to scale each of the numeric variables (x and z) to the unit interval. I store their minima and maxima in a separate matrix (the maximum is not simply max(x)). The first row is the min of each numeric variable, the second row is the max.
> cts.mat
x z
[1,] 1 -3
[2,] 4 5
How can I scale the x and z columns using the bounds in the matrix?
I tried something like
dat[, lapply(.SD, range01, cts.mat), .SDcol = c("x", "z")]
where range01 is the function
range01 <- function(x, cts.mat) {
x.as.string <- deparse(substitute(x))
# This is (x-lower)/(upper-lower)
(x - cts.mat[, x.as.string][1]) / (cts.mat[, x.as.string][2] - cts.mat[, x.as.string][1])
}
But this does not work. I think my core problem is that I don't know how to run an lapply with arguments that change for each column of dat. The changing arguments in this case are the min and max of each numeric column.
thanks for any help.

Answer 1

I found out a way to do it with a loop, which I think is good enough. If anyone knows how to do it with a data.table lapply, I'm still interested in a solution.
cts.names <- c("x", "z")
for (var in cts.names) {
dat[, (var) := scales::rescale(get(var),
from = c(0, 1),
to = cts.mat[, var])]
}

Answer 2

I'm very surprised by the lapply performance boost:
microbenchmark::microbenchmark(
set_loop={
for (var in colnames(cts.mat)) {
set(dat,j=var,value=scales::rescale(dat[[var]],
from = c(0, 1),
to = cts.mat[, var]))}
},
dt_loop={
for (var in colnames(cts.mat)) {
dat[, c(var) := scales::rescale(dat[[var]],
from = c(0, 1),
to = cts.mat[, var])]
}},
lapply={
lapply(colnames(cts.mat),
function(var) set(dat,j=var,value=scales::rescale(dat[[var]],
from = c(0, 1),
to = cts.mat[, var])))})
# Unit: microseconds
# expr min lq mean median uq max neval
# set_loop 2342.9 2380.90 2523.414 2437.15 2531.30 4856.3 100
# dt_loop 3109.0 3176.40 4000.359 3247.70 3383.35 64652.8 100
# lapply 65.8 74.85 103.510 83.85 90.00 2100.3 100
I verified that the results are the same with the three methods, interested if there is an explanation for this. Perhaps try ou with bigger data sets?
library(data.table)
dat <- read.table(text='x z y w
1: 3.307590 -1.66951137 b a
2: 1.809447 4.10331322 b b
3: 3.314621 3.69436879 a a
4: 1.896529 -0.08143017 c b
5: 3.317341 1.01839533 c a
6: 1.806456 -2.09547272 a b ',header=T)
setDT(dat)
cts.mat <- read.table(text='
x z
1: 1 -3
2: 4 5', header=T)
cts.mat <- as.matrix(cts.mat)
dat.ref <- copy(dat)
dat <- copy(dat.ref)
# set + loop
for (var in colnames(cts.mat)) {
set(dat,j=var,value=scales::rescale(dat[[var]],
from = c(0, 1),
to = cts.mat[, var]))}
result.set.loop <- copy(dat)
# dt + loop
dat <- copy(dat.ref)
for (var in colnames(cts.mat)) {
dat[, c(var) := scales::rescale(dat[[var]],
from = c(0, 1),
to = cts.mat[, var])]
}
result.dt.loop <- copy(dat)
# set + lapply
dat <- copy(dat.ref)
lapply(colnames(cts.mat),function(var) set(dat,j=var,value=scales::rescale(dat[[var]],
from = c(0, 1),
to = cts.mat[, var])))
#> [[1]]
#> x z y w
#> 1: 10.922770 -16.356091 b a
#> 2: 6.428341 29.826506 b b
#> 3: 10.943863 26.554950 a a
#> 4: 6.689587 -3.651441 c b
#> 5: 10.952023 5.147163 c a
#> 6: 6.419368 -19.763782 a b
#>
#> [[2]]
#> x z y w
#> 1: 10.922770 -16.356091 b a
#> 2: 6.428341 29.826506 b b
#> 3: 10.943863 26.554950 a a
#> 4: 6.689587 -3.651441 c b
#> 5: 10.952023 5.147163 c a
#> 6: 6.419368 -19.763782 a b
result.set.lapply <- copy(dat)
all.equal(result.dt.loop,result.set.loop)
#> [1] TRUE
all.equal(result.set.loop,result.set.lapply)
#> [1] TRUE