calculating difference between subsequent days without for loop - r

Question

I have the following data frame:
time <- c("2004-01-01 01:30:00","2004-01-01 04:30:00","2004-01-01 07:30:00",
"2004-01-01 10:30:00","2004-01-01 13:30:00","2004-01-01 16:30:00",
"2004-01-01 19:30:00","2004-01-01 22:30:00","2004-01-02 01:30:00",
"2004-01-02 04:30:00","2004-01-02 07:30:00","2004-01-02 10:30:00",
"2004-01-02 13:30:00","2004-01-02 16:30:00","2004-01-02 19:30:00",
"2004-01-02 22:30:00","2004-01-03 01:30:00","2004-01-03 04:30:00",
"2004-01-03 07:30:00","2004-01-03 10:30:00")
d <- c(0.00, 0.00,152808.30, 739872.84, 82641.22, 83031.04, 83031.04, 82641.22, 0.00,
0.00, 267024.71,1247414.7, 151638.85, 151249.03, 151249.03, 152028.67, 0.00, 0.00,
296650.81,1355783.85)
dat <- data.frame(time = time, dat = d)
which demonstrate the accumulation (per day) of solar radiation from a forecast model for 3 days.
To convert the units of solar radiation from J/m2 to W/m2, I need to calculate the difference between the different forecast times per day and divide by 10800 (the forecast time). Here is my attempt:
itime <- as.numeric(as.Date(dat$time))
utime <- unique(itime)
l <- list()
for(i in 1:length(utime)){
idx <- itime == utime[i]
dat2 <- dat[idx,]
dat3 <- dat2[1,2]/10800
for(ii in 2:nrow(dat2)){
dat3[ii] <- (abs(dat2[ii,2] - dat2[ii-1,2]))/10800
}
df <- data.frame(dateTime = dat2$time,
dd = dat3)
l[[i]] <- df
}
df1 <- do.call(rbind.data.frame, l)
df1[,1] <- as.POSIXct(df1[,1])
which performs as expected. However, the actual data on which I intend to use this code has a length of >100 days. Thus, it is not optimal to run a loop.
Is there another method I can use instead of a loop?
I have tried:
dat2 <- c(dat[1,2]/10800,rev(abs(diff(rev(dat[,2])))/10800))
df2 <- data.frame(time = as.POSIXct(dat[,1]), dd = dat2)
which gives nearly the same answer (as the loop), but it also calculates the difference between time steps in different days, instead of isolating the calculation to individual days.
plot(df1, type = 'l')
lines(df2, col = 'red')
As you can see, there is a mismatch during the early hours.
Can anyone suggest another method?

Answer 1

For your list l you can have the same result by
dat <- data.frame(
time = c("2004-01-01 01:30:00","2004-01-01 04:30:00","2004-01-01 07:30:00",
"2004-01-01 10:30:00","2004-01-01 13:30:00","2004-01-01 16:30:00",
"2004-01-01 19:30:00","2004-01-01 22:30:00","2004-01-02 01:30:00",
"2004-01-02 04:30:00","2004-01-02 07:30:00","2004-01-02 10:30:00",
"2004-01-02 13:30:00","2004-01-02 16:30:00","2004-01-02 19:30:00",
"2004-01-02 22:30:00","2004-01-03 01:30:00","2004-01-03 04:30:00",
"2004-01-03 07:30:00","2004-01-03 10:30:00"),
dat = c(0.00, 0.00,152808.30, 739872.84, 82641.22, 83031.04, 83031.04, 82641.22, 0.00,
0.00, 267024.71,1247414.7, 151638.85, 151249.03, 151249.03, 152028.67, 0.00, 0.00,
296650.81,1355783.85)
)
dat$itime <- as.numeric(as.Date(dat$time))
utime <- unique(dat$itime)
daydat <- function(u) {
dat2 <- dat[dat$itime==u,]
data.frame(dateTime = dat2$time, dd = c(dat2$dat[1], abs(diff(dat2$dat)))/10800)
}
l <- lapply(utime, daydat)
Here is a version with split():
dat$itime <- as.numeric(as.Date(dat$time))
daydat <- function(d) data.frame(dateTime = d$time, dd = c(d$dat[1], abs(diff(d$dat)))/10800)
L <- split(dat, dat$itime)
l <- lapply(L, daydat)
or without creating dat$itime:
daydat <- function(d) data.frame(dateTime = d$time, dd = c(d$dat[1], abs(diff(d$dat)))/10800)
l <- lapply(split(dat, as.Date(dat$time)), FUN=daydat)
or using by()
l2 <- unclass(by(dat, as.Date(dat$time), FUN=daydat))
If you want to have the result in the original dataframe you can use ave()
dat$dd <- ave(dat$dat, as.Date(dat$time), FUN=function(x) c(x[1], abs(diff(x)))/10800)

Answer 2

Use can use lag() from dplyr with group_by()
library(dplyr)
df <- dat %>%
mutate(date = as.Date(time)) %>%
group_by(date) %>%
mutate(before.dat = lag(dat, order_by=date)) %>%
mutate(diff = abs(dat - before.dat)/10800) %>%
select(time, date, dat, before.dat, diff)
df
#Source: local data frame [20 x 5]
#Groups: date [3]
# time date dat before.dat diff
# <fctr> <date> <dbl> <dbl> <dbl>
#1 2004-01-01 01:30:00 2004-01-01 0.00 NA NA
#2 2004-01-01 04:30:00 2004-01-01 0.00 0.00 0.00000000
#3 2004-01-01 07:30:00 2004-01-01 152808.30 0.00 14.14891667
#4 2004-01-01 10:30:00 2004-01-01 739872.84 152808.30 54.35782778
#5 2004-01-01 13:30:00 2004-01-01 82641.22 739872.84 60.85477963
#6 2004-01-01 16:30:00 2004-01-01 83031.04 82641.22 0.03609444
#7 2004-01-01 19:30:00 2004-01-01 83031.04 83031.04 0.00000000
#8 2004-01-01 22:30:00 2004-01-01 82641.22 83031.04 0.03609444
#9 2004-01-02 01:30:00 2004-01-02 0.00 NA NA
#10 2004-01-02 04:30:00 2004-01-02 0.00 0.00 0.00000000
#11 2004-01-02 07:30:00 2004-01-02 267024.71 0.00 24.72451019
#12 2004-01-02 10:30:00 2004-01-02 1247414.70 267024.71 90.77685093
#13 2004-01-02 13:30:00 2004-01-02 151638.85 1247414.70 101.46072685
#14 2004-01-02 16:30:00 2004-01-02 151249.03 151638.85 0.03609444
#15 2004-01-02 19:30:00 2004-01-02 151249.03 151249.03 0.00000000
#16 2004-01-02 22:30:00 2004-01-02 152028.67 151249.03 0.07218889
#17 2004-01-03 01:30:00 2004-01-03 0.00 NA NA
#18 2004-01-03 04:30:00 2004-01-03 0.00 0.00 0.00000000
#19 2004-01-03 07:30:00 2004-01-03 296650.81 0.00 27.46766759
#20 2004-01-03 10:30:00 2004-01-03 1355783.85 296650.81 98.06787407
Simplified code based on GGamba's comment
dat %>%
mutate(time = as.Date(time)) %>%
group_by(time) %>%
mutate(diff = (dat-lag(dat)) / 10800)