Subtract previous year's from value from each grouped row in data frame

Question

I am trying to calculated the lagged difference (or actual increase) for data that has been inadvertently aggregated. Each successive year in the data includes values from t

Answer 1

1) diff.zoo. With the zoo package its just a matter of converting it to zoo using split= and then performing the diff :

library(zoo)

zz <- zz0 <- read.zoo(df, split = "id", index = "year", FUN = identity)
zz[2:3, ] <- diff(zz)

It gives the following (in wide form rather than the long form you mentioned) where each column is an id and each row is a year minus the prior year:

> zz
   1  2  3  4  5
1  6  5  2  9  2
2 10  5 10  7 13
3  5 16 14 10 14

The wide form shown may actually be preferable but you can convert it to long form if you want that like this:

dt <- function(x) as.data.frame.table(t(x))
setNames(cbind(dt(zz), dt(zz0)[3]), c("id", "year", "value", "actual"))

This puts the years in ascending order which is the convention normally used in R.

2) rollapply. Also using zoo this alternative uses a rolling calculation to add the actual column to your data. It assumes the data is structured as you show with the same number of years in each group arranged in order:

df$actual <- rollapply(df$value, 6, partial = TRUE, align = "left",
   FUN = function(x) if (length(x) < 6) x[1] else x[1]-x[6])

3) subtraction. Making the same assumptions as in the prior solution we can further simplify it to just this which subtracts from each value the value 5 positions hence:

transform(df, actual = value - c(tail(value, -5), rep(0, 5)))

or this variation:

transform(df, actual = replace(value, year > 1, -diff(ts(value), 5)))

EDIT: added rollapply and subtraction solutions.

Answer 2

I think this will work for you. When you run into the diff problem just lengthen the vector by putting 0 in as the first number.

df <- df[order(df$id, df$year), ]
sdf <-split(df, df$id)
df$actual <- as.vector(sapply(seq_along(sdf), function(x) diff(c(0, sdf[[x]][,2]))))
df[order(as.numeric(rownames(df))),]

There's lots of ways to do this but this one is fairly fast and uses base.

Here's a second & third way of approaching this problem utilizing aggregate and by:

aggregate:

df <- df[order(df$id, df$year), ]
diff2 <- function(x) diff(c(0, x))
df$actual <- c(unlist(t(aggregate(value~id, df, diff2)[, -1])))
df[order(as.numeric(rownames(df))),]

by:

df <- df[order(df$id, df$year), ]
diff2 <- function(x) diff(c(0, x))
df$actual <- unlist(by(df$value, df$id, diff2))
df[order(as.numeric(rownames(df))),]

plyr

df <- df[order(df$id, df$year), ]
df <- data.frame(temp=1:nrow(df), df)
library(plyr)
df <- ddply(df, .(id), transform, actual=diff2(value))
df[order(-df$year, df$temp),][, -1]

It gives you the final product of:

> df[order(as.numeric(rownames(df))),]
   id value year actual
1   1    21    3      5
2   2    26    3     16
3   3    26    3     14
4   4    26    3     10
5   5    29    3     14
6   1    16    2     10
7   2    10    2      5
8   3    12    2     10
9   4    16    2      7
10  5    15    2     13
11  1     6    1      6
12  2     5    1      5
13  3     2    1      2
14  4     9    1      9
15  5     2    1      2

EDIT: Avoiding the Loop

May I suggest avoiding the loop and turning what I gave to you into a function (the by solution is the easiest one for me to work with) and sapply that to the two columns you desire.

set.seed(1234)  #make new data with another numeric column
x <- data.frame(id=1:5, value=sample(20:30, 5, replace=T), year=3)
y <- data.frame(id=1:5, value=sample(10:19, 5, replace=T), year=2)
z <- data.frame(id=1:5, value=sample(0:9, 5, replace=T), year=1)
df <- rbind(x, y, z)
df <- df.rep <- data.frame(df[, 1:2], new.var=df[, 2]+sample(1:5, nrow(df), 
          replace=T), year=df[, 3])


df <- df[order(df$id, df$year), ]
diff2 <- function(x) diff(c(0, x))                   #function one
group.diff<- function(x) unlist(by(x, df$id, diff2)) #answer turned function
df <- data.frame(df, sapply(df[, 2:3], group.diff))  #apply group.diff to col 2:3
df[order(as.numeric(rownames(df))),]                 #reorder it

Of course you'd have to rename these unless you used transform as in:

df <- df[order(df$id, df$year), ]
diff2 <- function(x) diff(c(0, x))                   #function one
group.diff<- function(x) unlist(by(x, df$id, diff2)) #answer turned function
df <- transform(df, actual=group.diff(value), actual.new=group.diff(new.var))   
df[order(as.numeric(rownames(df))),]

This would depend on how many variables you were doing this to.

Answer 3

Kind of hackish but keeping in place your wonderful Reduce you could add mock rows to your df for year 0:

mockRows <- data.frame(id = 1:5, value = 0, year = 0)
(df <- rbind(df, mockRows))
(df <- df[order(df$id, df$year), ])

(diffs <- lapply(split(df, df$id), function(x){diff(x$value)}))
(df <- df[df$year != 0,])

(df$actual <- Reduce(c, diffs)) # flatten the list of lists
df[order(as.numeric(rownames(df))),]

This is the output:

   id value year actual
1   1    21    3      5
2   2    26    3     16
3   3    26    3     14
4   4    26    3     10
5   5    29    3     14
6   1    16    2     10
7   2    10    2      5
8   3    12    2     10
9   4    16    2      7
10  5    15    2     13
11  1     6    1      6
12  2     5    1      5
13  3     2    1      2
14  4     9    1      9
15  5     2    1      2