Difference by Group using dplyr and data.table - r

Question

I want to calculate difference by groups. Although I referred R: Function “diff” over various groups thread on SO, for unknown reason, I am unable to find the difference. I have tried three methods : a) spread b) dplyr::mutate with base::diff() c) data.table with base::diff(). While a) works, I am unsure how I can solve this problem using b) and c).
Description about the data:
I have revenue data for the product by year. I have categorized years >= 2013 as Period 2 (called P2), and years < 2013 as Period 1 (called P1).
Sample data:
dput(Test_File)
structure(list(Ship_Date = c(2010, 2010, 2012, 2012, 2012, 2012,
2017, 2017, 2017, 2016, 2016, 2016, 2011, 2017), Name = c("Apple",
"Apple", "Banana", "Banana", "Banana", "Banana", "Apple", "Apple",
"Apple", "Banana", "Banana", "Banana", "Mango", "Pineapple"),
Revenue = c(5, 10, 13, 14, 15, 16, 25, 25, 25, 1, 2, 4, 5,
7)), .Names = c("Ship_Date", "Name", "Revenue"), row.names = c(NA,
14L), class = "data.frame")
Expected Output
dput(Diff_Table)
structure(list(Name = c("Apple", "Banana", "Mango", "Pineapple"
), P1 = c(15, 58, 5, NA), P2 = c(75, 7, NA, 7), Diff = c(60,
-51, NA, NA)), .Names = c("Name", "P1", "P2", "Diff"), class = "data.frame", row.names = c(NA,
-4L))
Here's my code:
Method 1: Using spread [Works]
data.table::setDT(Test_File)
cutoff<-2013
Test_File[Test_File$Ship_Date>=cutoff,"Ship_Period"]<-"P2"
Test_File[Test_File$Ship_Date<cutoff,"Ship_Period"]<-"P1"
Diff_Table<- Test_File %>%
dplyr::group_by(Ship_Period,Name) %>%
dplyr::mutate(Revenue = sum(Revenue)) %>%
dplyr::select(Ship_Period, Name,Revenue) %>%
dplyr::ungroup() %>%
dplyr::distinct() %>%
tidyr::spread(key = Ship_Period,value = Revenue) %>%
dplyr::mutate(Diff = `P2` - `P1`)
Method 2: Using dplyr [Doesn't work: NAs are generated in Diff column.]
Diff_Table<- Test_File %>%
dplyr::group_by(Ship_Period,Name) %>%
dplyr::mutate(Revenue = sum(Revenue)) %>%
dplyr::select(Ship_Period, Name,Revenue) %>%
dplyr::ungroup() %>%
dplyr::distinct() %>%
dplyr::arrange(Name,Ship_Period, Revenue) %>%
dplyr::group_by(Ship_Period,Name) %>%
dplyr::mutate(Diff = diff(Revenue))
Method 3: Using data.table [Doesn't work: It generates all zeros in Diff column.]
Test_File[,Revenue1 := sum(Revenue),by=c("Ship_Period","Name")]
Diff_Table<-Test_File[,.(Diff = diff(Revenue1)),by=c("Ship_Period","Name")]
Question: Can someone please help me with method 2 and method 3 above? I am fairly new to R so I apologize if my work sounds too basic. I am still learning this language.

Answer 1

We can do this with data.table. Convert the 'data.frame' to 'data.table' (setDT(Test_File)), grouped by the run-length-id of 'Name' and 'Name', get the sum of 'Revenue', reshape it to 'wide' format with dcast, get the difference between 'P2' and 'P1' and assign (:=) it to 'Diff'
library(data.table)
dcast(setDT(Test_File)[, .(Revenue = sum(Revenue)),
.(grp=rleid(Name), Name)], Name~ paste0("P", rowid(Name)),
value.var = "Revenue")[, Diff := P2 - P1][]
# Name P1 P2 Diff
#1: Apple 15 75 60
#2: Banana 58 7 -51
#3: Mango 5 NA NA
#4: Pineapple 7 NA NA
Or for third case, i.e. base R, we create a grouping column based on whether the adjacent elements in 'Name' are the same or not ('grp'), then aggregate the 'Revenue' by 'Name' and 'grp' to find the sum, create a sequence column, reshape it to 'wide' and transform the dataset to create the 'Diff' column
Test_File$grp <- with(Test_File, cumsum(c(TRUE, Name[-1]!=Name[-length(Name)])))
d1 <- aggregate(Revenue~Name +grp, Test_File, sum)
d1$Seq <- with(d1, ave(seq_along(Name), Name, FUN = seq_along))
transform(reshape(d1[-2], idvar = "Name", timevar = "Seq",
direction = "wide"), Diff = Revenue.2- Revenue.1)
The tidyverse method can also be done using
library(dplyr)
library(tidyr)
Test_File %>%
group_by(grp = cumsum(c(TRUE, Name[-1]!=Name[-length(Name)])), Name) %>%
summarise(Revenue = sum(Revenue)) %>%
group_by(Name) %>%
mutate(Seq = paste0("P", row_number())) %>%
select(-grp) %>%
spread(Seq, Revenue) %>%
mutate(Diff = P2-P1)
#Source: local data frame [4 x 4]
#Groups: Name [4]
# Name P1 P2 Diff
# <chr> <dbl> <dbl> <dbl>
#1 Apple 15 75 60
#2 Banana 58 7 -51
#3 Mango 5 NA NA
#4 Pineapple 7 NA NA
Update
Based on the OP's comments to use only diff function
library(data.table)
setDT(Test_File)[, .(Revenue = sum(Revenue)), .(Name, grp = rleid(Name))
][, .(P1 = Revenue[1L], P2 = Revenue[2L], Diff = diff(Revenue)) , Name]
# Name P1 P2 Diff
#1: Apple 15 75 60
#2: Banana 58 7 -51
#3: Mango 5 NA NA
#4: Pineapple 7 NA NA
Or with dplyr
Test_File %>%
group_by(grp = cumsum(c(TRUE, Name[-1]!=Name[-length(Name)])), Name) %>%
summarise(Revenue = sum(Revenue)) %>%
group_by(Name) %>%
summarise(P1 = first(Revenue), P2 = last(Revenue)) %>%
mutate(Diff = P2-P1)

Answer 2

This will do:
library("data.table")
setDT(Test_File)
T <- Test_File[, .(P=sum(Revenue)),by=.(Ship_Date, Name)]
T[Ship_Date>=2013][T[Ship_Date<2013][CJ(Name=T$Name, unique=TRUE), on="Name"], on="Name"][,`:=`(P1=i.P, P2=P, Diff=P-i.P)][]
# Ship_Date Name P i.Ship_Date i.P P1 P2 Diff
# 1: 2017 Apple 75 2010 15 15 75 60
# 2: 2016 Banana 7 2012 58 58 7 -51
# 3: NA Mango NA 2011 5 5 NA NA
# 4: 2017 Pineapple 7 NA NA NA 7 NA
Or with only the wanted columns:
T[Ship_Date>=2013][T[Ship_Date<2013][CJ(Name=T$Name, unique=TRUE), on="Name"], on="Name"][,`:=`(P1=i.P, P2=P, Diff=P-i.P)][,.(Name, P1, P2, Diff)]
# Name P1 P2 Diff
# 1: Apple 15 75 60
# 2: Banana 58 7 -51
# 3: Mango 5 NA NA
# 4: Pineapple NA 7 NA
Here is a variant using setnames():
setnames(T[Ship_Date>=2013][T[Ship_Date<2013][CJ(Name=T$Name, unique=TRUE), on="Name"], on="Name"],
c("P", "i.P"), c("P2", "P1"))[, Diff:=P2-P1][]