Combine two dataframes one above the other - r

I have two dataframes and I want to put one above the other "with" column names of second as a row of the new dataframe. Column names are different and one dataframe has more columns.
For example:
mydf1 <- data.frame(V1=c(1:5), V2=c(21:25))
mydf1
V1 V2
1 1 21
2 2 22
3 3 23
4 4 24
5 5 25
mydf2 <- data.frame(C1=c(1:10), C2=c(21:30),C3=c(41:50))
mydf2
C1 C2 C3
1 1 21 41
2 2 22 42
3 3 23 43
4 4 24 44
5 5 25 45
6 6 26 46
7 7 27 47
8 8 28 48
9 9 29 49
10 10 30 50
Result:
mydf
V1 V2
1 1 21 NA
2 2 22 NA
3 3 23 NA
4 4 24 NA
5 5 25 NA
6 C1 C2 C3
7 1 21 41
8 2 22 42
9 3 23 43
10 4 24 44
11 5 25 45
12 6 26 46
13 7 27 47
14 8 28 48
15 9 29 49
16 10 30 50
I dont care if all numeric values treated like characters.
Many thanks

You can do this easily without any packages:
mydf1 <- data.frame(V1=c(1:5), V2=c(21:25))
mydf1[,3] <- NA
names(mydf1) <- c("one", "two", "three")
mydf2 <- data.frame(C1=c(1:10), C2=c(21:30),C3=c(41:50))
names <- t(as.data.frame(names(mydf2)))
names <- as.data.frame(names)
names(mydf2) <- c("one", "two", "three")
names(names) <- c("one", "two", "three")
mydf3 <- rbind(mydf1, names)
mydf4 <- rbind(mydf3, mydf2)
> mydf4
one two three
1 1 21 <NA>
2 2 22 <NA>
3 3 23 <NA>
4 4 24 <NA>
5 5 25 <NA>
6 C1 C2 C3
7 1 21 41
8 2 22 42
9 3 23 43
10 4 24 44
11 5 25 45
12 6 26 46
13 7 27 47
14 8 28 48
15 9 29 49
16 10 30 50
>
Of course, you can edit the <- c("one", "two", "three") to make the final column names whatever you'd like. For example:
> mydf1 <- data.frame(V1=c(1:5), V2=c(21:25))
> mydf1[,3] <- NA
> names(mydf1) <- c("V1", "V2", "NA")
> mydf2 <- data.frame(C1=c(1:10), C2=c(21:30),C3=c(41:50))
> names <- t(as.data.frame(names(mydf2)))
> names <- as.data.frame(names)
> names(mydf2) <- c("V1", "V2", "NA")
> names(names) <- c("V1", "V2", "NA")
> mydf3 <- rbind(mydf1, names)
> mydf4 <- rbind(mydf3, mydf2)
> row.names(mydf4) <- NULL
> mydf4
V1 V2 NA
1 1 21 <NA>
2 2 22 <NA>
3 3 23 <NA>
4 4 24 <NA>
5 5 25 <NA>
6 C1 C2 C3
7 1 21 41
8 2 22 42
9 3 23 43
10 4 24 44
11 5 25 45
12 6 26 46
13 7 27 47
14 8 28 48
15 9 29 49
16 10 30 50
If you need to resort a package for any reason when scaling this up to your real use case, then try melt from reshape2 or the package plyr. However, use of a package shouldn't be necessary.

I don't know what you tried with write.table, but that seems to me like the way to go.
I would create a function something like this:
myFun <- function(...) {
L <- list(...)
temp <- tempfile()
maxCol <- max(vapply(L, ncol, 1L))
lapply(L, function(x)
suppressWarnings(
write.table(x, file = temp, row.names = FALSE,
sep = ",", append = TRUE)))
read.csv(temp, header = FALSE, fill = TRUE,
col.names = paste0("New_", sequence(maxCol)),
stringsAsFactors = FALSE)
}
Usage would then simply be:
myFun(mydf1, mydf2)
# New_1 New_2 New_3
# 1 V1 V2
# 2 1 21
# 3 2 22
# 4 3 23
# 5 4 24
# 6 5 25
# 7 C1 C2 C3
# 8 1 21 41
# 9 2 22 42
# 10 3 23 43
# 11 4 24 44
# 12 5 25 45
# 13 6 26 46
# 14 7 27 47
# 15 8 28 48
# 16 9 29 49
# 17 10 30 50
The function is written such that you can specify more than two data.frames as input:
mydf3 <- data.frame(matrix(1:8, ncol = 4))
myFun(mydf1, mydf2, mydf3)
# New_1 New_2 New_3 New_4
# 1 V1 V2
# 2 1 21
# 3 2 22
# 4 3 23
# 5 4 24
# 6 5 25
# 7 C1 C2 C3
# 8 1 21 41
# 9 2 22 42
# 10 3 23 43
# 11 4 24 44
# 12 5 25 45
# 13 6 26 46
# 14 7 27 47
# 15 8 28 48
# 16 9 29 49
# 17 10 30 50
# 18 X1 X2 X3 X4
# 19 1 3 5 7
# 20 2 4 6 8

Here's one approach with the rbind.fill function (part of the plyr package).
library(plyr)
setNames(rbind.fill(setNames(mydf1, names(mydf2[seq(mydf1)])),
rbind(names(mydf2), mydf2)), names(mydf1))
V1 V2 NA
1 1 21 <NA>
2 2 22 <NA>
3 3 23 <NA>
4 4 24 <NA>
5 5 25 <NA>
6 C1 C2 C3
7 1 21 41
8 2 22 42
9 3 23 43
10 4 24 44
11 5 25 45
12 6 26 46
13 7 27 47
14 8 28 48
15 9 29 49
16 10 30 50

Give this a try.
Assign the column names from the second data set to a vector, and then replace the second set's names with the names from the first set. Then create a list where the middle element is the vector you assigned. Now when you call rbind, it should be fine since everything is in the right order.
d1$V3 <- NA
nm <- names(d2)
names(d2) <- names(d1)
dc <- do.call(rbind, list(d1,nm,d2))
rownames(dc) <- NULL
dc

Related

R: How to merge a new data frame to several other data frames in a list

I have several seperate data frames that I would like to keep separated because merging them together would create a very large element.
However, there are variables from another data frame that I would like to merge with all of them now.
Here is an example of what I would like to do:
df1 <- data.frame(ID1 = c(1:10), Var1 = rep(c(1,0),5))
df2 <- data.frame(ID1 = c(1:10), Var2 = c(21:30))
dfs <- Filter(function(x) is(x, "data.frame"), mget(ls()))
mergewith <- data.frame(ID1 = c(1:10), ID2 = c(41:50))
My goal is that df1 and df2 will look like this:
df1
ID1 Var1 ID2
1 1 1 41
2 2 0 42
3 3 1 43
4 4 0 44
5 5 1 45
6 6 0 46
7 7 1 47
8 8 0 48
9 9 1 49
10 10 0 50
df2
ID1 Var2 ID2
1 1 21 41
2 2 22 42
3 3 23 43
4 4 24 44
5 5 25 45
6 6 26 46
7 7 27 47
8 8 28 48
9 9 29 49
10 10 30 50
What I have tried so far is:
dat = lapply(dfs,function(x){
merge(names(x), mergewith, by = "ID1");x})
list2env(dat,.GlobalEnv)
However, then I get the following message:
"'by' must specify a uniquely valid column"
Is it possible to do this without using a loop?
You can try Map
> Map(function(x, y) merge(x, y, by = "ID1"), dfs, list(mergewith))
[[1]]
ID1 Var1 ID2
1 1 1 41
2 2 0 42
3 3 1 43
4 4 0 44
5 5 1 45
6 6 0 46
7 7 1 47
8 8 0 48
9 9 1 49
10 10 0 50
[[2]]
ID1 Var2 ID2
1 1 21 41
2 2 22 42
3 3 23 43
4 4 24 44
5 5 25 45
6 6 26 46
7 7 27 47
8 8 28 48
9 9 29 49
10 10 30 50
You can use lapply to merge all the dataframes in dfs with mergewith. Use list2env to get the changed dataframes in the global environment.
list2env(lapply(dfs, function(x) merge(x, mergewith, by = 'ID1')), .GlobalEnv)

Assign new value to database based on value stored in another database

Here I share with you a simplified version of my issue. Say I have 6 observations (pid) for two variables:
pid <- c(1,2,3,4,5,6)
V1 <- c(11,11,33,11,22,33)
V2 <- c("A", "C", "M", "M", "A", "A")
data <- data.frame(pid, V1, V2)
# pid V1 V2
# 1 1 11 A
# 2 2 11 C
# 3 3 33 M
# 4 4 11 M
# 5 5 22 A
# 6 6 33 A
I would like to create a new column based on the values associated to the different combinations I have of V1 and V2, that stored in a second database:
V1 <- c(11,11,11,22,22,22,33,33,33)
V2 <- c("A", "C", "M","A", "C", "M","A", "C", "M")
valueA <- c(16,26,36,46,56,66,76,86,96)
valueB <- c(15,25,35,45,55,65,75,85,95)
values <- data.frame(V1, V2, valueA, valueB)
# V1 V2 valueA valueB
# 1 11 A 16 15
# 2 11 C 26 25
# 3 11 M 36 35
# 4 22 A 46 45
# 5 22 C 56 55
# 6 22 M 66 65
# 7 33 A 76 75
# 8 33 C 86 85
# 9 33 M 96 95
I tried this, following #akrun suggestion:
data <- mutate (data,
valueA = as.integer (ifelse(data$V1 %in% values$V1
& data$V2 %in% values$V2, values$valueA, NA))
)
But the result is the following:
# pid V1 V2 valueA
# 1 1 11 A 16
# 2 2 11 C 26
# 3 3 33 M 36
# 4 4 11 M 46
# 5 5 22 A 56
# 6 6 33 A 66
As you can see, the combination 33 M is 36 while it should be 96...
I would like to archive this:
# pid V1 V2 valueA
# 1 1 11 A 16
# 2 2 11 C 26
# 3 3 33 M 96
# 4 4 11 M 36
# 5 5 22 A 46
# 6 6 33 A 76
any suggestions on how to fix this? Any help would me much appreciated!
I solved the issue above creating a single column merging V1 and V2 as follows:
data$unique <- paste(data$V1,data$V2)
values$unique <- paste(values$V1, values$V2)
and then merged by the new column:
merge(x = data, y = values, by = "unique")
# unique pid V1.x V2.x V1.y V2.y valueA valueB
# 1 11 A 1 11 A 11 A 16 15
# 2 11 C 2 11 C 11 C 26 25
# 3 11 M 4 11 M 11 M 36 35
# 4 22 A 5 22 A 22 A 46 45
# 5 33 A 6 33 A 33 A 76 75
# 6 33 M 3 33 M 33 M 96 95

tidyverse: binding list elements efficiently

I want to bind data.frames of same number of rows from a list as given below.
df1 <- data.frame(A1 = 1:10, B1 = 11:20)
df2 <- data.frame(A1 = 1:10, C1 = 21:30)
df3 <- data.frame(A2 = 1:15, B2 = 11:25, C2 = 31:45)
df4 <- data.frame(A2 = 1:15, D2 = 11:25, E2 = 51:65)
df5 <- 5
ls <- list(df1, df2, df3, df4, df5)
library(dplyr)
#>
#> Attaching package: 'dplyr'
#> The following objects are masked from 'package:stats':
#>
#> filter, lag
#> The following objects are masked from 'package:base':
#>
#> intersect, setdiff, setequal, union
bind_cols(ls[1], ls[2], .id = NULL)
#> New names:
#> * A1 -> A1...1
#> * A1 -> A1...3
#> A1...1 B1 A1...3 C1
#> 1 1 11 1 21
#> 2 2 12 2 22
#> 3 3 13 3 23
#> 4 4 14 4 24
#> 5 5 15 5 25
#> 6 6 16 6 26
#> 7 7 17 7 27
#> 8 8 18 8 28
#> 9 9 19 9 29
#> 10 10 20 10 30
bind_cols(ls[3], ls[4], .id = NULL)
#> New names:
#> * A2 -> A2...1
#> * A2 -> A2...4
#> A2...1 B2 C2 A2...4 D2 E2
#> 1 1 11 31 1 11 51
#> 2 2 12 32 2 12 52
#> 3 3 13 33 3 13 53
#> 4 4 14 34 4 14 54
#> 5 5 15 35 5 15 55
#> 6 6 16 36 6 16 56
#> 7 7 17 37 7 17 57
#> 8 8 18 38 8 18 58
#> 9 9 19 39 9 19 59
#> 10 10 20 40 10 20 60
#> 11 11 21 41 11 21 61
#> 12 12 22 42 12 22 62
#> 13 13 23 43 13 23 63
#> 14 14 24 44 14 24 64
#> 15 15 25 45 15 25 65
In my actual list, I have about twenty data.frames of different number of rows. I wonder if there is a more efficient way of binding data.frames of same number of rows without giving the name and index of list elements.
It is easier to do this by splitting. Create a grouping index with gl
grp <- as.integer(gl(length(ls), 2, length(ls)))
and then use split
library(dplyr)
library(purrr)
library(stringr)
split(ls, grp) %>% # // split by the grouping index
map(bind_cols) %>% # // loop over the `list` and use `bind_cols`
set_names(str_c('df', seq_along(.))) %>% # // name the `list`
list2env(.GlobalEnv) # // create objects in global env
-output
head(df1)
# A1...1 B1 A1...3 C1
#1 1 11 1 21
#2 2 12 2 22
#3 3 13 3 23
#4 4 14 4 24
#5 5 15 5 25
#6 6 16 6 26
head(df2)
# A2...1 B2 C2 A2...4 D2 E2
#1 1 11 31 1 11 51
#2 2 12 32 2 12 52
#3 3 13 33 3 13 53
#4 4 14 34 4 14 54
#5 5 15 35 5 15 55
#6 6 16 36 6 16 56
head(df3)
# A tibble: 1 x 1
# ...1
# <dbl>
#1 5
NOTE:
It is better to keep the elements in the list instead of creating objects in the global environment i.e. list2env
ls is a function name and naming an object with function name is not a good option as it can lead to buggy situations
Maybe not the optimal approach but you can use a loop and bind the dataframes with same number of columns into a new dataframes. The main of this code is to check the dimension of each dataframe and create an unique vector. Then in the loop you can use lapply() to subset the dataframes in ls and the bind their columns. Here the code (Updated considering the little df5, you can make the trick managing it as a dataframe):
library(dplyr)
#Data
df1 <- data.frame(A1 = 1:10, B1 = 11:20)
df2 <- data.frame(A1 = 1:10, C1 = 21:30)
df3 <- data.frame(A2 = 1:15, B2 = 11:25, C2 = 31:45)
df4 <- data.frame(A2 = 1:15, D2 = 11:25, E2 = 51:65)
df5 <- 5
#List
ls <- list(df1, df2, df3, df4,df5)
#Index
index <- sapply(ls,function(x)dim(as.data.frame(x))[1])
m <- unique(index)
#Loop
for(i in 1:length(m))
{
assign(paste0('df',i),do.call(bind_cols,ls[lapply(ls,function(x) dim(as.data.frame(x))[1]==m[i])==T]))
}
Output:
df1
A1...1 B1 A1...3 C1
1 1 11 1 21
2 2 12 2 22
3 3 13 3 23
4 4 14 4 24
5 5 15 5 25
6 6 16 6 26
7 7 17 7 27
8 8 18 8 28
9 9 19 9 29
10 10 20 10 30
df2
A2...1 B2 C2 A2...4 D2 E2
1 1 11 31 1 11 51
2 2 12 32 2 12 52
3 3 13 33 3 13 53
4 4 14 34 4 14 54
5 5 15 35 5 15 55
6 6 16 36 6 16 56
7 7 17 37 7 17 57
8 8 18 38 8 18 58
9 9 19 39 9 19 59
10 10 20 40 10 20 60
11 11 21 41 11 21 61
12 12 22 42 12 22 62
13 13 23 43 13 23 63
14 14 24 44 14 24 64
15 15 25 45 15 25 65
df3
...1
1 5

Modify data frames in list to have same # of rows

I'm trying to combine data frames (hundreds of them), but they have different numbers of rows.
df1 <- data.frame(c(7,5,3,4,5), c(43,56,23,78,89))
df2 <- data.frame(c(7,5,3,4,5,8,5), c(43,56,23,78,89,45,78))
df3 <- data.frame(c(7,5,3,4,5,8,5,6,7), c(43,56,23,78,89,45,78,56,67))
colnames(df1) <- c("xVar1","xVar2")
colnames(df2) <- c("yVar1","yVar2")
colnames(df3) <- c("zVar1","zVar2")
a1 <- list(df1,df2,df3)
a1 is what is my initial data actually looks like when I get it.
Now if I do:
b1 <- as.data.frame(a1)
I get an error, because the # of rows is not the same in the data (this would work fine if the # of rows was the same).
How do I make the # of rows equal or work around this issue?
I would like to be able to merge the data in this way (here is a working example with the same # of rows):
df1b <- data.frame(c(7,5,3,4,5), c(43,56,23,78,89))
df2b <- data.frame(c(7,5,3,4,6), c(43,56,24,48,89))
df3b <- data.frame(c(7,5,3,4,5), c(43,56,23,78,89))
colnames(df1b) <- c("xVar1","xVar2")
colnames(df2b) <- c("yVar1","yVar2")
colnames(df3b) <- c("zVar1","zVar2")
a2 <- list(df1b,df2b,df3b)
b2 <- as.data.frame(a2)
Thanks!
cbind.fill from rowr provides functionality for this and fills missing elements with NA:
library(purrr)
library(rowr)
b1 <- purrr::reduce(a1,cbind.fill,fill=NA)
One can add a key (row count as variable value in this case) to each dataframe then merge by the key.
# get list of dfs (should prob import data into a list of dfs instead)
list_df<-mget(ls(pattern = "df[0-9]"))
#add newcolumn -- "key"
list_df<-lapply(list_df, function(df, newcol) {
df[[newcol]]<-seq(nrow(df))
return(df)
}, "key")
#merge function
MergeAllf <- function(x, y){
df <- merge(x, y, by= "key", all.x= T, all.y= T)
}
#pass list to merge funct
library(tidyverse)
data <- Reduce(MergeAllf, list_df)%>%
select(key, everything())#reorder or can drop "key"
data
key xVar1 xVar2 yVar1 yVar2 zVar1 zVar2
1 1 7 43 7 43 7 43
2 2 5 56 5 56 5 56
3 3 3 23 3 23 3 23
4 4 4 78 4 78 4 78
5 5 5 89 5 89 5 89
6 6 NA NA 8 45 8 45
7 7 NA NA 5 78 5 78
8 8 NA NA NA NA 6 56
9 9 NA NA NA NA 7 67
Solution 1
You can achieve this with rbindlist(). Note that the column names will be the column names of the first data frame in the list:
library(data.table)
b1 = data.frame(rbindlist(a1))
> b1
xVar1 xVar2
1 7 43
2 5 56
3 3 23
4 4 78
5 5 89
6 7 43
7 5 56
8 3 23
9 4 78
10 5 89
11 8 45
12 5 78
13 7 43
14 5 56
15 3 23
16 4 78
17 5 89
18 8 45
19 5 78
20 6 56
21 7 67
Solution 2
Alternatively, you make all the columns have the same name, then bind by row:
b1 = lapply(a1, setNames, c("Var1","Var2"))
Now you can bind by rows:
b1 = do.call(dplyr::bind_rows, b1)
> b1
Var1 Var2
1 7 43
2 5 56
3 3 23
4 4 78
5 5 89
6 7 43
7 5 56
8 3 23
9 4 78
10 5 89
11 8 45
12 5 78
13 7 43
14 5 56
15 3 23
16 4 78
17 5 89
18 8 45
19 5 78
20 6 56
21 7 67

Add data frames row wise with [d]plyr

I have two data frames
df1
# a b
# 1 10 20
# 2 11 21
# 3 12 22
# 4 13 23
# 5 14 24
# 6 15 25
df2
# a b
# 1 4 8
I want the following output:
df3
# a b
# 1 14 28
# 2 15 29
# 3 16 30
# 4 17 31
# 5 18 32
# 6 19 33
i.e. add df2 to each row of df1.
Is there a way to get the desired output using plyr (mdplyr??) or dplyr?
I see no reason for "dplyr" for something like this. In base R you could just do:
df1 + unclass(df2)
# a b
# 1 14 28
# 2 15 29
# 3 16 30
# 4 17 31
# 5 18 32
# 6 19 33
Which is the same as df1 + list(4, 8).
One liner with dplyr.
mutate_each(df1, funs(.+ df2$.), a:b)
# a b
#1 14 28
#2 15 29
#3 16 30
#4 17 31
#5 18 32
#6 19 33
A base R solution using sweet function sweep:
sweep(df1, 2, unlist(df2), '+')
# a b
#1 14 28
#2 15 29
#3 16 30
#4 17 31
#5 18 32
#6 19 33

Resources