Resampling from a large data.frame - r

Question

I have a large data.frame of this structure:
min.reps <- 1
max.reps <- 3
set.seed(1)
df <- do.call(rbind,lapply(1:100, function(i) {
reps <- seq(1,as.integer(runif(1,min.reps, max.reps)), 1)
vals <- runif(length(reps), 0, 100)
return(data.frame(id=rep(i,length(reps)),rep=reps,val=vals,stringsAsFactors=F))
}))
head(df)
id rep val
1 1 1 37.212390
2 2 1 90.820779
3 2 2 20.168193
4 3 1 94.467527
5 3 2 66.079779
6 4 1 6.178627
Each df$id has between min.reps and max.reps observations (df$val). In reality instead of 100 ids I have ~5,000,000 ids.
For each df$id I'd like to add one more value, sampled from a normal distribution with mean and sd as the median and mad over its existing values, respectively.
This is trivial to do this way:
add.reps <- 1
all.ids <- unique(df$id)
require(dplyr)
new.df <- do.call(rbind, lapply(all.ids, function(i) {
id.df <- dplyr::filter(df, id == i)
add.df <- rbind(id.df, data.frame(id = rep(i,add.reps), rep = max(id.df$rep) + add.reps, val = rnorm(add.reps, median(id.df$val), mad(id.df$val)), stringsAsFactors = F))
}))
But I'm wondering if there's a much faster way to achieve this given the dimensions of my real data.frame.

Answer 1

This should be much faster:
add.reps <- 1
do.call(rbind, lapply(split(df, df$id), function(x) rbind(x,
data.frame(id = rep(unique(x$id), add.reps), rep = max(x$rep) + add.reps,
val = rnorm(add.reps, median(x$val), mad(x$val)), stringsAsFactors = F))))

Answer 2

Ok, so so far:
require(microbenchmark)
microbenchmark(
new.df <- do.call(rbind, lapply(all.ids, function(i) {
id.df <- dplyr::filter(df, id == i)
add.df <- rbind(id.df, data.frame(id = rep(i,add.reps), rep = max(id.df$rep) + add.reps, val = rnorm(add.reps, median(id.df$val), mad(id.df$val)), stringsAsFactors = F))
}))
)
new.df <- do.call(rbind, lapply(all.ids, function(i) { id.df <- dplyr::filter(df, id == i) add.df <- rbind(id.df, data.frame(id = rep(i, add.reps), rep = max(id.df$rep) + add.reps, val = rnorm(add.reps, median(id.df$val), mad(id.df$val)), stringsAsFactors = F)) }))
min lq mean median uq max neval
212.9906 225.1345 371.9314 260.9686 332.5619 1621.586 100
vs.
microbenchmark(
new.df <- do.call(rbind, lapply(split(df, df$id), function(x) rbind(x,
data.frame(id = rep(unique(x$id), add.reps), rep = max(x$rep) + add.reps,
val = rnorm(add.reps, median(x$val), mad(x$val)), stringsAsFactors = F))))
)
new.df <- do.call(rbind, lapply(split(df, df$id), function(x) rbind(x, data.frame(id = rep(unique(x$id), add.reps), rep = max(x$rep) + add.reps, val = rnorm(add.reps, median(x$val), mad(x$val)), stringsAsFactors = F))))
min lq mean median uq max neval
133.8357 135.1846 202.9654 137.2722 160.5121 1401.03 100
I wonder if this can still be further improved