I have two data frames as follows:
df1<- read.table( text="id time group class income
12 NA NA NA NA
17 NA NA NA NA
19 NA NA NA NA
36 NA NA NA NA
14 NA NA NA NA
15 NA NA NA NA
8 NA NA NA NA
22 NA NA NA NA
33 NA NA NA NA
11 NA NA NA NA
",h=T)
The second data frame is as follows:
df2<- read.table( text="id time group class income age
17 3 A 1 2 12
11 6 A 3 12 11
36 9 E 2 11 23
19 19 E 2 13 13
8 23 M 2 14 14
15 11 M 1 16 13
12 3 P 1 15 15
14 3 Q 2 13 13
33 4 Z 3 19 13
22 5 G 3 20 11",h=T)
I want to get the following table using ids
df3<- read.table( text="id time group class income
12 23 P 1 15
17 3 A 1 2
19 3 E 2 13
36 5 E 2 11
14 3 Q 2 13
15 3 M 1 16
8 23 M 2 14
22 5 G 3 20
33 4 Z 3 19
11 6 A 3 12",h=T)
I am sorry, but I was unsuccessful to find some possible solutions to show my attempt. I appreciate your help.
Here's a dplyr solution assuming you want the order of id from df1 with most of the data from df2:
library(dplyr)
df1 %>%
select(id) %>%
left_join(df2) %>%
select(-age)
Joining, by = "id"
id time group class income
1 12 3 P 1 15
2 17 3 A 1 2
3 19 19 E 2 13
4 36 9 E 2 11
5 14 3 Q 2 13
6 15 11 M 1 16
7 8 23 M 2 14
8 22 5 G 3 20
9 33 4 Z 3 19
10 11 6 A 3 12
Related
Using reduce(bind_cols), the list elements of same dimension may be combined. However, I would like to know how to combine only same dimension (may be specified dimesion in some way) elements from a list which may have elements of different dimension.
library(tidyverse)
df1 <- data.frame(A1 = 1:10, A2 = 10:1)
df2 <- data.frame(B = 11:30)
df3 <- data.frame(C = 31:40)
ls1 <- list(df1, df3)
ls1
[[1]]
A1 A2
1 1 10
2 2 9
3 3 8
4 4 7
5 5 6
6 6 5
7 7 4
8 8 3
9 9 2
10 10 1
[[2]]
C
1 31
2 32
3 33
4 34
5 35
6 36
7 37
8 38
9 39
10 40
ls1 %>%
reduce(bind_cols)
A1 A2 C
1 1 10 31
2 2 9 32
3 3 8 33
4 4 7 34
5 5 6 35
6 6 5 36
7 7 4 37
8 8 3 38
9 9 2 39
10 10 1 40
ls2 <- list(df1, df2, df3)
ls2
[[1]]
A1 A2
1 1 10
2 2 9
3 3 8
4 4 7
5 5 6
6 6 5
7 7 4
8 8 3
9 9 2
10 10 1
[[2]]
B
1 11
2 12
3 13
4 14
5 15
6 16
7 17
8 18
9 19
10 20
11 21
12 22
13 23
14 24
15 25
16 26
17 27
18 28
19 29
20 30
[[3]]
C
1 31
2 32
3 33
4 34
5 35
6 36
7 37
8 38
9 39
10 40
ls2 %>%
reduce(bind_cols)
Error: Can't recycle `..1` (size 10) to match `..2` (size 20).
Run `rlang::last_error()` to see where the error occurred.
Question
Looking for a function to combine all data.frames in a list with an argument of number of rows.
One option could be:
map(split(lst, map_int(lst, NROW)), bind_cols)
$`10`
A1 A2 C
1 1 10 31
2 2 9 32
3 3 8 33
4 4 7 34
5 5 6 35
6 6 5 36
7 7 4 37
8 8 3 38
9 9 2 39
10 10 1 40
$`20`
B
1 11
2 12
3 13
4 14
5 15
6 16
7 17
8 18
9 19
10 20
11 21
12 22
13 23
14 24
15 25
16 26
17 27
18 28
19 29
20 30
You can use -
n <- 1:max(sapply(ls2, nrow))
res <- do.call(cbind, lapply(ls2, `[`, n, ,drop = FALSE))
res
# A1 A2 B C
#1 1 10 11 31
#2 2 9 12 32
#3 3 8 13 33
#4 4 7 14 34
#5 5 6 15 35
#6 6 5 16 36
#7 7 4 17 37
#8 8 3 18 38
#9 9 2 19 39
#10 10 1 20 40
#NA NA NA 21 NA
#NA.1 NA NA 22 NA
#NA.2 NA NA 23 NA
#NA.3 NA NA 24 NA
#NA.4 NA NA 25 NA
#NA.5 NA NA 26 NA
#NA.6 NA NA 27 NA
#NA.7 NA NA 28 NA
#NA.8 NA NA 29 NA
#NA.9 NA NA 30 NA
A little-bit shorter with purrr::map_dfc
purrr::map_dfc(ls2, `[`, n, , drop = FALSE)
We can use cbind.fill from rowr
library(rowr)
do.call(cbind.fill, c(ls2, fill = NA))
A base R option using tapply + sapply
tapply(
ls2,
sapply(ls2, nrow),
function(x) do.call(cbind, x)
)
gives
$`10`
A1 A2 C
1 1 10 31
2 2 9 32
3 3 8 33
4 4 7 34
5 5 6 35
6 6 5 36
7 7 4 37
8 8 3 38
9 9 2 39
10 10 1 40
$`20`
B
1 11
2 12
3 13
4 14
5 15
6 16
7 17
8 18
9 19
10 20
11 21
12 22
13 23
14 24
15 25
16 26
17 27
18 28
19 29
20 30
You may also use if inside reduce if you want to combine similar elements of list (case: when first item in list has priority)
df1 <- data.frame(A1 = 1:10, A2 = 10:1)
df2 <- data.frame(B = 11:30)
df3 <- data.frame(C = 31:40)
ls1 <- list(df1, df3)
ls2 <- list(df1, df2, df3)
library(tidyverse)
reduce(ls2, ~if(nrow(.x) == nrow(.y)){bind_cols(.x, .y)} else {.x})
#> A1 A2 C
#> 1 1 10 31
#> 2 2 9 32
#> 3 3 8 33
#> 4 4 7 34
#> 5 5 6 35
#> 6 6 5 36
#> 7 7 4 37
#> 8 8 3 38
#> 9 9 2 39
#> 10 10 1 40
Created on 2021-06-09 by the reprex package (v2.0.0)
Here's another tidyverse option.
We're creating a dummy ID in each data.frame based on the row_number(), then joining all data.frames by the dummy ID, and then dropping the dummy ID.
ls2 %>%
map(., ~mutate(.x, id = row_number())) %>%
reduce(full_join, by = "id") %>%
select(-id)
This gives us:
A1 A2 B C
1 1 10 11 31
2 2 9 12 32
3 3 8 13 33
4 4 7 14 34
5 5 6 15 35
6 6 5 16 36
7 7 4 17 37
8 8 3 18 38
9 9 2 19 39
10 10 1 20 40
11 NA NA 21 NA
12 NA NA 22 NA
13 NA NA 23 NA
14 NA NA 24 NA
15 NA NA 25 NA
16 NA NA 26 NA
17 NA NA 27 NA
18 NA NA 28 NA
19 NA NA 29 NA
20 NA NA 30 NA
We can also use Reduce function from base R:
lst <- list(df1, df2, df3)
# First we create id number for each underlying data set
lst |>
lapply(\(x) {x$id <- 1:nrow(x);
x
}
) -> ls2
Reduce(function(x, y) if(nrow(x) == nrow(y)){
merge(x, y, by = "id")
} else {
x
}, ls2)
id A1 A2 C
1 1 1 10 31
2 2 2 9 32
3 3 3 8 33
4 4 4 7 34
5 5 5 6 35
6 6 6 5 36
7 7 7 4 37
8 8 8 3 38
9 9 9 2 39
10 10 10 1 40
Given a dataset which contains a grouping variable and a column of integers which is incomplete (contains NAs) and the beginning and ending integer vary by group and the length of each group varies (and could be NA). How might one fill in the NA integer values by completing the sequence.
The following dataset may be used as an example:
library(dplyr)
set.seed(5112021)
dat1 <- bind_rows(data.frame(Group=1,Seq=(3:20)),
data.frame(Group=2,Seq=(-1:25))) %>%
mutate(rn = rnorm(45,mean=0.5,sd=1),
Seq = ifelse(rn < 0.4,NA,Seq)) %>%
select(-rn) %>%
group_by(Group) %>%
mutate(Seq = ifelse(Seq==-1,NA,Seq))
dat1
Group Seq
1 1 NA
2 1 NA
3 1 NA
4 1 6
5 1 7
6 1 8
7 1 NA
8 1 10
9 1 11
10 1 NA
11 1 13
12 1 NA
13 1 15
14 1 NA
15 1 NA
16 1 NA
17 1 NA
18 1 20
19 2 NA
20 2 0
21 2 NA
22 2 2
23 2 3
24 2 NA
25 2 5
26 2 6
27 2 7
28 2 8
29 2 NA
30 2 10
31 2 NA
32 2 12
33 2 NA
34 2 NA
35 2 NA
36 2 16
37 2 17
38 2 NA
39 2 NA
40 2 NA
41 2 NA
42 2 22
43 2 NA
44 2 NA
45 2 NA
One way to do this could be to make use of row_numbers (since they are a sequence of integers) by group and calculate the difference between the non-missing values and the row number (which is a unique value) and then add that value back to the row number.
for example
dat2 <- dat1 %>%
group_by(Group) %>%
mutate(rn = row_number(),
diff = mean(Seq-rn,na.rm=T)) %>%
mutate(New_Seq = rn+diff) %>%
select(-rn,-diff)
dat2
Group Seq New_Seq
1 1 NA 3
2 1 NA 4
3 1 NA 5
4 1 6 6
5 1 7 7
6 1 8 8
7 1 NA 9
8 1 10 10
9 1 11 11
10 1 NA 12
11 1 13 13
12 1 NA 14
13 1 15 15
14 1 NA 16
15 1 NA 17
16 1 NA 18
17 1 NA 19
18 1 20 20
19 2 NA -1
20 2 0 0
21 2 NA 1
22 2 2 2
23 2 3 3
24 2 NA 4
25 2 5 5
26 2 6 6
27 2 7 7
28 2 8 8
29 2 NA 9
30 2 10 10
31 2 NA 11
32 2 12 12
33 2 NA 13
34 2 NA 14
35 2 NA 15
36 2 16 16
37 2 17 17
38 2 NA 18
39 2 NA 19
40 2 NA 20
41 2 NA 21
42 2 22 22
43 2 NA 23
44 2 NA 24
45 2 NA 25
While this works, it doesn't seem very elegant and may be slow for very large datasets with many grouping variables. I'm curiouse if there is a more 'Tidyverse' way to do this.
You could do something like:
df %>%
group_by(Group) %>%
mutate(newseq = seq_along(Group) + (first(na.omit(Seq)) - sum(cumall(is.na(Seq)))) - 1) %>%
ungroup()
Or
df %>%
group_by(Group) %>%
mutate(newseq = seq(first(na.omit(Seq)) - sum(cumall(is.na(Seq))), length.out = n())) %>%
ungroup()
Or
df %>%
group_by(Group) %>%
mutate(newseq = 0:(n() - 1) + (first(na.omit(Seq)) - sum(cumall(is.na(Seq))))) %>%
ungroup()
All these do the same thing: shift the start of the sequence by the difference of the first non-NA value and the number of NAs before it.
Output
Group Seq newseq
<int> <int> <dbl>
1 1 NA 3
2 1 NA 4
3 1 NA 5
4 1 6 6
5 1 7 7
6 1 8 8
7 1 NA 9
8 1 10 10
9 1 11 11
10 1 NA 12
# ... with 35 more rows
First create row number, then take the max difference of Seq and row_number and add to row number:
dat1 %>%
group_by(Group) %>%
mutate(rn = row_number(),
Seq = rn + max(Seq - rn, na.rm = TRUE)) %>%
ungroup() %>%
select(-rn)
Output:
Group Seq
<dbl> <int>
1 1 3
2 1 4
3 1 5
4 1 6
5 1 7
6 1 8
7 1 9
8 1 10
9 1 11
10 1 12
11 1 13
12 1 14
13 1 15
14 1 16
15 1 17
16 1 18
17 1 19
18 1 20
19 2 -1
20 2 0
21 2 1
22 2 2
23 2 3
24 2 4
25 2 5
26 2 6
27 2 7
28 2 8
29 2 9
30 2 10
31 2 11
32 2 12
33 2 13
34 2 14
35 2 15
36 2 16
37 2 17
38 2 18
39 2 19
40 2 20
# … with 5 more rows
data:
set.seed(5112021)
dat1 <- bind_rows(data.frame(Group=1,Seq=(3:20)),
data.frame(Group=2,Seq=(-1:25))) %>%
mutate(rn = rnorm(45,mean=0.5,sd=1),
Seq = ifelse(rn < 0.4,NA,Seq)) %>%
select(-rn) %>%
group_by(Group) %>%
mutate(Seq = ifelse(Seq==-1,NA,Seq))
I would like to merge two datasets like this:
data1 <- data.frame (id=paste("id",1:10, sep=""), a=1:10, b= 11:20, d=21:30)
data2 <- data.frame(id=paste("id", 11:20, sep=""), a=1:10, b=11:20, e= 21:30)
The merged data will have all common columns and merged by column "id".
id a b d e
1 1 1 11 21 NA
2 2 2 12 22 NA
3 3 3 13 23 NA
4 4 4 14 24 NA
5 5 5 15 25 NA
6 6 6 16 26 NA
7 7 7 17 27 NA
8 8 8 18 28 NA
9 9 9 19 29 NA
10 10 10 20 30 NA
11 11 1 11 NA 21
12 12 2 12 NA 22
13 13 3 13 NA 23
14 14 4 14 NA 24
15 15 5 15 NA 25
16 16 6 16 NA 26
17 17 7 17 NA 27
18 18 8 18 NA 28
19 19 9 19 NA 29
20 20 10 20 NA 30
You can see the missing columns are added and values are set to NA. The manual work around would like this, but looking for elegant way to do in R which a large number of variables are mismatching :
data1$e=NA
data2$d = NA
rbind(data1, data2)
I'd just use dplyr::bind_rows() instead of rbind(). That'll give you the behavior you want with no additional lifting.
data1 <- data.frame (id=1:10, a=1:10, b= 11:20, d=21:30)
data2 <- data.frame(id=11:20, a=1:10, b=11:20, e= 21:30)
dplyr::bind_rows(data1, data2)
#> id a b d e
#> 1 1 1 11 21 NA
#> 2 2 2 12 22 NA
#> 3 3 3 13 23 NA
#> 4 4 4 14 24 NA
#> 5 5 5 15 25 NA
#> 6 6 6 16 26 NA
#> 7 7 7 17 27 NA
#> 8 8 8 18 28 NA
#> 9 9 9 19 29 NA
#> 10 10 10 20 30 NA
#> 11 11 1 11 NA 21
#> 12 12 2 12 NA 22
#> 13 13 3 13 NA 23
#> 14 14 4 14 NA 24
#> 15 15 5 15 NA 25
#> 16 16 6 16 NA 26
#> 17 17 7 17 NA 27
#> 18 18 8 18 NA 28
#> 19 19 9 19 NA 29
#> 20 20 10 20 NA 30
Created on 2021-03-25 by the reprex package (v1.0.0)
Here is the code for my example dataset.
df = data.frame("group" =c(rep(1,5),rep(1,6),rep(2,4),rep(2,3)), "time" = c(rep(NA,5),seq(1,6),rep(NA,4),seq(1,3)), "p" = seq(1,18) )
group time p
1 1 NA 1
2 1 NA 2
3 1 NA 3
4 1 NA 4
5 1 NA 5
6 1 1 6
7 1 2 7
8 1 3 8
9 1 4 9
10 1 5 10
11 1 6 11
12 2 NA 12
13 2 NA 13
14 2 NA 14
15 2 NA 15
16 2 1 16
17 2 2 17
18 2 3 18
I would like to figure out how to apply a function by group to only the values that have time then append the result as a new column in the data frame. Here is my example function I would like to apply.
pfunc <- function(p){
p+5
}
The output I am hoping to obtain would look as follows.
group time p new_p
1 1 NA 1 NA
2 1 NA 2 NA
3 1 NA 3 NA
4 1 NA 4 NA
5 1 NA 5 NA
6 1 1 6 11
7 1 2 7 12
8 1 3 8 13
9 1 4 9 14
10 1 5 10 15
11 1 6 11 16
12 2 NA 12 NA
13 2 NA 13 NA
14 2 NA 14 NA
15 2 NA 15 NA
16 2 1 16 21
17 2 2 17 22
18 2 3 18 23
You can try this:
library(dplyr)
df %>% group_by(group) %>%
mutate(pnew=ifelse(is.na(time),time,time+5))
# A tibble: 18 x 4
# Groups: group [2]
group time p pnew
<dbl> <int> <int> <dbl>
1 1 NA 1 NA
2 1 NA 2 NA
3 1 NA 3 NA
4 1 NA 4 NA
5 1 NA 5 NA
6 1 1 6 6
7 1 2 7 7
8 1 3 8 8
9 1 4 9 9
10 1 5 10 10
11 1 6 11 11
12 2 NA 12 NA
13 2 NA 13 NA
14 2 NA 14 NA
15 2 NA 15 NA
16 2 1 16 6
17 2 2 17 7
18 2 3 18 8
Update
You can use this function:
increase <- function(data,n)
{
data %>% group_by(group) %>%
mutate(pnew=ifelse(is.na(time),time,time+n)) -> result
return(result)
}
increase(df,n = 10)
# A tibble: 18 x 4
# Groups: group [2]
group time p pnew
<dbl> <int> <int> <dbl>
1 1 NA 1 NA
2 1 NA 2 NA
3 1 NA 3 NA
4 1 NA 4 NA
5 1 NA 5 NA
6 1 1 6 11
7 1 2 7 12
8 1 3 8 13
9 1 4 9 14
10 1 5 10 15
11 1 6 11 16
12 2 NA 12 NA
13 2 NA 13 NA
14 2 NA 14 NA
15 2 NA 15 NA
16 2 1 16 11
17 2 2 17 12
18 2 3 18 13
Update 2
I hope this helps:
df %>% group_by(group) %>% rowwise() %>% mutate(pnew=ifelse(is.na(time),NA,pfunc(time)))
# A tibble: 18 x 4
# Rowwise: group
group time p pnew
<dbl> <int> <int> <dbl>
1 1 NA 1 NA
2 1 NA 2 NA
3 1 NA 3 NA
4 1 NA 4 NA
5 1 NA 5 NA
6 1 1 6 6
7 1 2 7 7
8 1 3 8 8
9 1 4 9 9
10 1 5 10 10
11 1 6 11 11
12 2 NA 12 NA
13 2 NA 13 NA
14 2 NA 14 NA
15 2 NA 15 NA
16 2 1 16 6
17 2 2 17 7
18 2 3 18 8
I would like to repeat the first two rows for each id two times. I don't know how to do that. Does anyone have a suggestion?
id <- rep(1:4,each=6)
scored <- c(12,13,NA,NA,NA,NA,14,20,NA,NA,NA,NA,23,56,NA,NA,NA,NA, 45,78,NA,NA,NA,NA)
df <- data.frame(id,scored)
df
id scored
1 1 12
2 1 13
3 1 NA
4 1 NA
5 1 NA
6 1 NA
7 2 14
8 2 20
9 2 NA
10 2 NA
11 2 NA
12 2 NA
13 3 23
14 3 56
15 3 NA
16 3 NA
17 3 NA
18 3 NA
19 4 45
20 4 78
21 4 NA
22 4 NA
23 4 NA
24 4 NA
>
I want it to look like:
df
id score
1 1 12
2 1 13
3 1 12
4 1 13
5 1 12
6 1 13
7 2 14
8 2 20
9 2 14
10 2 20
11 2 14
12 2 20
13 3 23
14 3 56
15 3 23
16 3 56
17 3 23
18 3 56
19 4 45
20 4 78
21 4 45
22 4 78
23 4 45
24 4 78
>
..................................................
..................................................
..................................................
We can do a group by rep on the non-NA elements of 'scored'
library(dplyr)
df %>%
group_by(id) %>%
mutate(scored = rep(scored[!is.na(scored)], length.out = n()))
# A tibble: 24 x 2
# Groups: id [4]
# id scored
# <int> <dbl>
# 1 1 12
# 2 1 13
# 3 1 12
# 4 1 13
# 5 1 12
# 6 1 13
# 7 2 14
# 8 2 20
# 9 2 14
#10 2 20
# … with 14 more rows