How to first group_by() then iterate lm() through columns? - r

Question

Let's say we have a data frame with a set of 3 dependent variables and 6 independent variables tagged by a grouping variable. An example of this format is generated with the sample code below:
library(tidyverse)
library(broom)
n <- 15
df <- data.frame(groupingvar= sample(letters[1:2], size = n, replace = TRUE),
y1 = rnorm(n,10,1), y2=rnorm(n,100,10), y3=rnorm(n,1000,100),
x1= rnorm(n,10,1), x2=rnorm(n,10,1), x3=rnorm(n,10,1),
x4=rnorm(n,10,1), x5=rnorm(n,10,1), x6=rnorm(n,10,1))
df <- arrange(df,groupingvar)
If I wanted to regress each of the y1, y2, y3 on the set of x1 through x6 I could use something along the lines of:
y <- as.matrix(select(df,y1:y3))
x <- as.matrix(select(df,x1:x6))
regs <-lm(y~x)
coeffs <- tidy(regs)
coeffs <- arrange(coeffs,response, term)
(by making use of the following line from the lm() help: "If response is a matrix, a linear model is fitted separately by least-squares to each column of the matrix.")
However, if I need to first group by the grouping variable and then apply the lm function then I'm not quite sure how to do it. I have tried the following, but it produces the same set of coefficients for both groups.
regs2 <- df %>% group_by(groupingvar) %>%
do(fit2 = lm(as.matrix(select(df,y1:y3)) ~ as.matrix(select(df,x1:x6))))
coeffs2 <- tidy(regs2,fit2)
coeffs2 <- arrange(coeffs2,groupingvar, response)

Answer 1

In data.table, you could melt (reshape long -- stack the outcome variables in one column instead of stored in three columns) & lm by both groupingvar and the outcome variable:
library(data.table)
setDT(df)
#alternatively, set id.vars = c('groupingvar', paste0('x', 1:6)), etc.
longDT = melt(df, id.vars = grep('y', names(df), invert = TRUE))
#this helper function basically splits a named vector into
# its two components
coefsplit = function(reg) {
beta = coef(reg)
list(var = names(beta), coef = beta)
}
#I personally wouldn't assign longDT, I'd just chain this onto
# the output of melt;
longDT[ , coefsplit(lm(value ~ ., data = .SD)), by = .(groupingvar, variable)]
# groupingvar variable var coef
# 1: a y1 (Intercept) -3.595564e+03
# 2: a y1 x1 -3.796627e+01
# 3: a y1 x2 -1.557268e+02
# 4: a y1 x3 2.862738e+02
# 5: a y1 x4 1.579548e+02
# ...
# 38: b y3 x2 2.136253e+01
# 39: b y3 x3 -3.810176e+01
# 40: b y3 x4 4.187719e+01
# 41: b y3 x5 -2.586184e+02
# 42: b y3 x6 1.181879e+02
# groupingvar variable var coef

Answer 2

I also found a way to achieve this using cbind() as follows:
library(tidyverse)
library(broom)
n <- 20
df4 <- data.frame(groupingvar= sample(1:2, size = n, replace = TRUE),
y1 = rnorm(n,10,1), y2=rnorm(n,100,10), y3=rnorm(n,1000,100),
x1= rnorm(n,10,1), x2=rnorm(n,10,1), x3=rnorm(n,10,1),
x4=rnorm(n,10,1), x5=rnorm(n,10,1), x6=rnorm(n,10,1))
df4 <- arrange(df4,groupingvar)
regs <- df4 %>% group_by(groupingvar) %>%
do(fit = lm(cbind(y1,y2,y3) ~ . -groupingvar, data = .))
coeffs <- tidy(regs, fit)