I have the following sample data:
d <- data.frame(id=c(1,1,1,2,2), time=c(1,1,1,1,1), var=runif(5))
id time var
1 1 1 0.373448545
2 1 1 0.007007124
3 1 1 0.840572603
4 2 1 0.684893481
5 2 1 0.822581501
I want to reshape this data.frame to wide format using dcast such that the output is the following:
id var.1 var.2 var.3
1 1 0.3734485 0.007007124 0.8405726
2 2 0.6848935 0.822581501 NA
Does anyone has some ideas?
Create a sequence column, seq, by id and then use dcast:
library(reshape2)
set.seed(123)
d <- data.frame(id=c(1,1,1,2,2), time=c(1,1,1,1,1), var=runif(5))
d2 <- transform(d, seq = ave(id, id, FUN = seq_along))
dcast(d2, id ~ seq, value.var = "var")
giving:
id 1 2 3
1 1 0.28758 0.78831 0.40898
2 2 0.88302 0.94047 NaN
A dplyr/tidyr option with spread would be
library(dplyr)
library(tidyr)
d %>%
group_by(id) %>%
mutate(n1= paste0("var.",row_number())) %>%
spread(n1, var) %>%
select(-time)
# id var.1 var.2 var.3
# (int) (dbl) (dbl) (dbl)
#1 1 0.3734485 0.007007124 0.8405726
#2 2 0.6848935 0.822581501 NA
Ok - here's a working solution. The key is to add a counting variable. My solution for this is a bit complicated - maybe you can come up with something better.
library(dplyr)
library(magrittr)
library(reshape2)
d <- data.frame(id=c(1,1,1,2,2,3,3,3,3), time=c(1,1,1,1,1,1,1,1,1), var=runif(9))
group_by(d, id) %>%
summarise(n = n()) %>%
data.frame() -> count
f <- c()
for (i in 1:nrow(count)) {
f <- c(f, 1:count$n[i])
}
d <- data.frame(d, f)
dcast(d, id ~ f, value.var = "var")
Related
My data looks like this:
id
date
1
a
1
a
1
b
1
c
1
c
1
c
2
z
2
z
2
e
2
x
I want to calculate the average of duplicates per id i.e for id=1 we have 2a 1b 3c I want the output to be 2.
The result shoulbe like this:
id
mean
1
2
2
1.333
You can use mean(table(date)) to get average of counts, apply it by for each id value.
Using dplyr -
library(dplyr)
df %>%
group_by(id) %>%
summarise(mean = mean(table(date)))
# id mean
# <int> <dbl>
#1 1 2
#2 2 1.33
Or with base R aggregate.
aggregate(date~id, df, function(x) mean(table(x)))
You can try a tidyverse
library(tidyverse)
d %>%
group_by(id) %>%
count(date) %>%
summarise(mean = mean(n))
# A tibble: 2 x 2
id mean
<int> <dbl>
1 1 2
2 2 1.33
Using base R you can try
foo <- function(x) mean(rle(x)$length)
aggregate(d$date, by=list(d$id), foo)
The data
d <- read.table(text ="id date
1 a
1 a
1 b
1 c
1 c
1 c
2 a
2 a
2 e
2 z", header=T)
using data.table package
library(data.table)
# dt <- your_data_frame %>% as.data.table() ## convert to table from frame
dt[, .(N=.N), by = .(id,date)][, .(mean = mean(N)), by = id]
Another data.table option
> setDT(df)[, .(Mean = .N / uniqueN(date)), id]
id Mean
1: 1 2.000000
2: 2 1.333333
or
dcast(setDT(df), id ~ date, fill = NA)[, .(Mean = rowMeans(.SD, na.rm = TRUE)), id]
gives
id Mean
1: 1 2.000000
2: 2 1.333333
We can use
library(dplyr)
df1 %>%
group_by(id) %>%
summarise(Mean = count(cur_data(), date) %>%
pull(n) %>%
mean)
A table approach using the base package:
at<-table(a$id,a$date)
apply(at,1,function(x) sum(x)/sum(x!=0))
# 1 2
#2.000000 1.333333
The dataset:
a = data.frame('id'=c(1,1,1,1,1,1,2,2,2,2),'date'=c('a','a','b','c','c','c','a','a','e','z'))
here is a package free solution
a = cbind(c(1,1,1,1,1,1,2,2,2,2),c('a','a','b','c','c','c','a','a','e','z'))
b = matrix(ncol = 2)[-1,]
for(i in unique(a[,1])){
b=rbind(b,c(i,sum(table(a[a[,1]==i,2]))/length(table(a[a[,1]==i,2]))))
}
The output:
[,1] [,2]
[1,] "1" "2"
[2,] "2" "1.33333333333333"
I am trying to obtain counts of a certain categorical variable in 2 separate columns, with each column reflecting the presence or an absence of an indicator variable. This is for a very large data frame. Here is an example data frame to further illustrate what I'm trying to do.
X <- (1:10)
Y <- c('a','b','a','c','b','b','a','a','c','c')
Z <- c(0,1,1,1,0,1,0,1,1,1)
test_df <- data.frame(X,Y,Z)
I would like to make a new DF grouped by 'a','b', and 'c' with 2 columns to the right, one with counts of the letter for Z==1 and the a count of that letter for Z==0.
The dplyr way:
library(dplyr)
library(tidyr)
#Code
res <- test_df %>% group_by(Y,Z) %>% summarise(N=n()) %>%
pivot_wider(names_from = Z,values_from=N,
values_fill = 0)
Output:
# A tibble: 3 x 3
# Groups: Y [3]
Y `0` `1`
<chr> <int> <int>
1 a 2 2
2 b 1 2
3 c 0 3
We can use values_fn in pivot_wider to do this in a single step
library(dplyr)
library(tidyr)
test_df %>%
pivot_wider(names_from = Z, values_from = X,
values_fn = length, values_fill = 0)
# A tibble: 3 x 3
# Y `0` `1`
# <chr> <int> <int>
#1 a 2 2
#2 b 1 2
#3 c 0 3
A base R option using aggregate + reshape
replace(
u <- reshape(
aggregate(X ~ ., test_df, length),
idvar = "Y",
timevar = "Z",
direction = "wide"
),
is.na(u),
0
)
giving
Y X.0 X.1
1 a 2 2
2 b 1 2
5 c 0 3
One way with data.table:
library(data.table)
setDT(test_df)
test_df[ , z1 := sum(Z==1), by=Y]
test_df[ , z0 := sum(Z==0), by=Y]
In base R you can use table :
table(test_df$Y, test_df$Z)
# 0 1
# a 2 2
# b 1 2
# c 0 3
There is my problem that I can't solve it:
Data:
df <- data.frame(f1=c("a", "a", "b", "b", "c", "c", "c"),
v1=c(10, 11, 4, 5, 0, 1, 2))
data.frame:f1 is factor
f1 v1
a 10
a 11
b 4
b 5
c 0
c 1
c 2
# What I want is:(for example, fetch data with the number of element of some level == 2, then to data.frame)
a b
10 4
11 5
Thanks in advance!
I might be missing something simple here , but the below approach using dplyr works.
library(dplyr)
nlevels = 2
df1 <- df %>%
add_count(f1) %>%
filter(n == nlevels) %>%
select(-n) %>%
mutate(rn = row_number()) %>%
spread(f1, v1) %>%
select(-rn)
This gives
# a b
# <int> <int>
#1 10 NA
#2 11 NA
#3 NA 4
#4 NA 5
Now, if you want to remove NA's we can do
do.call("cbind.data.frame", lapply(df1, function(x) x[!is.na(x)]))
# a b
#1 10 4
#2 11 5
As we have filtered the dataframe which has only nlevels observations, we would have same number of rows for each column in the final dataframe.
split might be useful here to split df$v1 into parts corresponding to df$f1. Since you are always extracting equal length chunks, it can then simply be combined back to a data.frame:
spl <- split(df$v1, df$f1)
data.frame(spl[lengths(spl)==2])
# a b
#1 10 4
#2 11 5
Or do it all in one call by combining this with Filter:
data.frame(Filter(function(x) length(x)==2, split(df$v1, df$f1)))
# a b
#1 10 4
#2 11 5
Here is a solution using unstack :
unstack(
droplevels(df[ave(df$v1, df$f1, FUN = function(x) length(x) == 2)==1,]),
v1 ~ f1)
# a b
# 1 10 4
# 2 11 5
A variant, similar to #thelatemail's solution :
data.frame(Filter(function(x) length(x) == 2, unstack(df,v1 ~ f1)))
My tidyverse solution would be:
library(tidyverse)
df %>%
group_by(f1) %>%
filter(n() == 2) %>%
mutate(i = row_number()) %>%
spread(f1, v1) %>%
select(-i)
# # A tibble: 2 x 2
# a b
# * <dbl> <dbl>
# 1 10 4
# 2 11 5
or mixing approaches :
as_tibble(keep(unstack(df,v1 ~ f1), ~length(.x) == 2))
Using all base functions (but you should use tidyverse)
# Add count of instances
x$len <- ave(x$v1, x$f1, FUN = length)
# Filter, drop the count
x <- x[x$len==2, c('f1','v1')]
# Hacky pivot
result <- data.frame(
lapply(unique(x$f1), FUN = function(y) x$v1[x$f1==y])
)
colnames(result) <- unique(x$f1)
> result
a b
1 10 4
2 11 5
I'd like code this, may it helps for you
library(reshape2)
library(dplyr)
aa = data.frame(v1=c('a','a','b','b','c','c','c'),f1=c(10,11,4,5,0,1,2))
cc = aa %>% group_by(v1) %>% summarise(id = length((v1)))
dd= merge(aa,cc) #get the level
ee = dd[dd$aa==2,] #select number of level equal to 2
ee$id = rep(c(1,2),nrow(ee)/2) # reset index like (1,2,1,2)
dcast(ee, id~v1,value.var = 'f1')
all done!
I am trying to reshape data from long to wide format in R. I would like to get both counts of occurrences of a type variable by ID and sums of the values of a second variable (val) by ID and type as in the example below.
I was able to find answers for reshaping with either counts or sums but not for both simultaneously.
This is the original example data:
> df <- data.frame(id = c(1, 1, 1, 2, 2, 2),
+ type = c("A", "A", "B", "A", "B", "C"),
+ val = c(0, 1, 2, 0, 0, 4))
> df
id type val
1 1 A 0
2 1 A 1
3 1 B 2
4 2 A 0
5 2 B 0
6 2 C 4
The output I would like to obtain is the following:
id A.count B.count C.count A.sum B.sum C.sum
1 1 2 1 0 1 2 0
2 2 1 1 1 0 0 4
where the count columns display the number of occurrences of type A, B and C and the sum columns the sum of the values by type.
To achieve the counts I can, as suggested in this answer, use reshape2::dcast with the default aggregation function, length:
> require(reshape2)
> df.c <- dcast(df, id ~ type, value.var = "type", fun.aggregate = length)
> df.c
id A B C
1 1 2 1 0
2 2 1 1 1
Similarly, as suggested in this answer, I can also perform the reshape with the sums as output, this time using the sum aggregation function in dcast:
> df.s <- dcast(df, id ~ type, value.var = "val", fun.aggregate = sum)
> df.s
id A B C
1 1 1 2 0
2 2 0 0 4
I could merge the two:
> merge(x = df.c, y = df.s, by = "id", all = TRUE)
id A.x B.x C.x A.y B.y C.y
1 1 2 1 0 1 2 0
2 2 1 1 1 0 0 4
but is there a way of doing it all in one go (not necessarily with dcast or reshape2)?
From data.table v1.9.6, it is possible to cast multiple value.var columns and also cast by providing multiple fun.aggregate functions. See below:
library(data.table)
df <- data.table(df)
dcast(df, id ~ type, fun = list(length, sum), value.var = c("val"))
id val_length_A val_length_B val_length_C val_sum_A val_sum_B val_sum_C
1: 1 2 1 0 1 2 0
2: 2 1 1 1 0 0 4
Here is an approach with tidyverse
library(tidyverse)
df %>%
group_by(id, type) %>%
summarise(count = n(), Sum = sum(val)) %>%
gather(key, val, count:Sum) %>%
unite(typen, type, key, sep=".") %>%
spread(typen, val, fill = 0)
The data.table solution suggested is probably better but if you prefer using dcast and you have many value.var/fun.aggregate combinations, you could also do:
library(purrr)
cols <- c('type', 'val')
funs <- c(length, sum)
map2(cols, funs, ~ dcast(df, id~type, value.var = .x, fun.aggregate = .y)) %>%
reduce(left_join, by='id', suffix=c('.count', '.sum'))
I'll illustrate my question with an example.
Sample data:
df <- data.frame(ID = c(1, 1, 2, 2, 3, 5), A = c("foo", "bar", "foo", "foo", "bar", "bar"), B = c(1, 5, 7, 23, 54, 202))
df
ID A B
1 1 foo 1
2 1 bar 5
3 2 foo 7
4 2 foo 23
5 3 bar 54
6 5 bar 202
What I want to do is to summarize, by ID, the sum of B and the sum of B when A is "foo". I can do this in a couple steps like:
require(magrittr)
require(dplyr)
df1 <- df %>%
group_by(ID) %>%
summarize(sumB = sum(B))
df2 <- df %>%
filter(A == "foo") %>%
group_by(ID) %>%
summarize(sumBfoo = sum(B))
left_join(df1, df2)
ID sumB sumBfoo
1 1 6 1
2 2 30 30
3 3 54 NA
4 5 202 NA
However, I'm looking for a more elegant/faster way, as I'm dealing with 10gb+ of out-of-memory data in sqlite.
require(sqldf)
my_db <- src_sqlite("my_db.sqlite3", create = T)
df_sqlite <- copy_to(my_db, df)
I thought of using mutate to define a new Bfoo column:
df_sqlite %>%
mutate(Bfoo = ifelse(A=="foo", B, 0))
Unfortunately, this doesn't work on the database end of things.
Error in sqliteExecStatement(conn, statement, ...) :
RS-DBI driver: (error in statement: no such function: IFELSE)
You can do both sums in a single dplyr statement:
df1 <- df %>%
group_by(ID) %>%
summarize(sumB = sum(B),
sumBfoo = sum(B[A=="foo"]))
And here is a data.table version:
library(data.table)
dt = setDT(df)
dt1 = dt[ , .(sumB = sum(B),
sumBfoo = sum(B[A=="foo"])),
by = ID]
dt1
ID sumB sumBfoo
1: 1 6 1
2: 2 30 30
3: 3 54 0
4: 5 202 0
Writing up #hadley's comment as an answer
df_sqlite %>%
group_by(ID) %>%
mutate(Bfoo = if(A=="foo") B else 0) %>%
summarize(sumB = sum(B),
sumBfoo = sum(Bfoo)) %>%
collect
If you want to do counting instead of summarizing, then the answer is somewhat different. The change in code is small, especially in the conditional counting part.
df1 <- df %>%
group_by(ID) %>%
summarize(countB = n(),
countBfoo = sum(A=="foo"))
df1
Source: local data frame [4 x 3]
ID countB countBfoo
1 1 2 1
2 2 2 2
3 3 1 0
4 5 1 0
If you wanted to count the rows, instead of summing them, can you pass a variable to the function:
df1 <- df %>%
group_by(ID) %>%
summarize(RowCountB = n(),
RowCountBfoo = n(A=="foo"))
I get an error both with n() and nrow().