Apply subsetting or row deleting function columnwise, based on grouping - r

Question

Some sample data frame (real data has 500k observations by 20 variables):
set.seed(1)
dataframe <- data.frame()
IDs <- as.factor(sample(LETTERS[seq( from = 1, to = 3)], prob = c(0.2, 0.3, 0.5), 1000, replace = TRUE))
Var1 <- sample(x = c(20:1500), size = 1000, replace = TRUE)
Var2 <- sample(x = c(1:15), size = 1000, replace = TRUE)
Var3 <- sample(x = c(0.1:8.5), size = 1000, replace = TRUE)
Var4<- sample(x = c(12:255), size = 1000, replace = TRUE)
Var5 <- sample(x = c(14000000:15000000), size = 1000, replace = TRUE)
dataframe <- data.frame(IDs, Var1, Var2, Var3, Var4, Var5)
dataframe$Var5 <- as.POSIXlt(dataframe$Var5, origin = "1970-01-01")
For every subject in ID, I want to remove all rows for which Var1 are not within the range of (mean +/- 0.5 standard deviation) in Var1.
I guess the way to go is to use dplyr, pipe dataframe to group_by(ID), and apply a function. If so, I need help with both the function and dplyr commands.
My first attempt was to use a for loop with ID:
for(ID in levels(dataframe$IDs)){
# Get 0.5 standard deviations
sd05 <- sd(dataframe[which(dataframe$IDs == ID), "Var1"]) * 0.5
# Get mean for subsetting
mean_for_subset <- mean(dataframe[which(dataframe$IDs == ID), "Var1"])
dataframe[which( dataframe[which(dataframe$IDs == ID), "Var1"] > (mean_for_subset + sd05)
& dataframe[which(dataframe$IDs == ID), "Var1"] < (mean_for_subset - sd05))
,] <- NULL
}
That gives warnings as is.na() was not applied to vector or list and dataframe still has 1000 observations.

Answer 1

Using data.table:
library(data.table)
dataframe <- data.table(dataframe)
meanV1 <- dataframe[, mean(Var1)]
sdV1 <- 0.5 * dataframe[, sd(Var1)]
dataframe <- dataframe[Var1 < meanV1 + sdV1 & Var1 > meanV1 - sdV1]
Of if this is to be done by ID:
library(data.table)
dataframe <- data.table(dataframe)
dataframe[, c("mean1", "sd1") := list(mean(Var1), 0.5 * sd(Var1)), by = IDs]
dataframe <- dataframe[Var1 < mean1 + sd1 & Var1 > mean1 - sd1]
Then to remove the new rows:
dataframe[, c("mean1", "sd1") := NULL]
Done on two columns:
library(data.table)
dataframe <- data.table(dataframe)
dataframe[, c(
"mean1",
"sd1",
"mean2",
"sd2"
) := list(
mean(Var1),
0.5 * sd(Var1)),
mean(Var2),
0.5 * sd(Var2)),
by = IDs
]
dataframe <- dataframe[
Var1 < mean1 + sd1 &
Var1 > mean1 - sd1 &
Var2 < mean2 + sd2 &
Var2 > mean2 - sd2
]
dataframe[, c("mean1", "sd1", "mean2", "sd2") := NULL]