I am trying to convert the following list to a dataframe.
I have tried melt/cast, ldply, unlist etc but can't seem to get the expected output.
Many thanks in advance!
df <- list(
name=rep(c(11,12), each=1),
value=rnorm(2),
name=rep(c(13,14), each=1),
value=rnorm(2)
)
df
I want the following output in a dataframe:
name value
11 1.187
12 0.691
13 0.452
14 0.898
An option is to stack into a two column data.frame, and spread it back to 'wide' format
library(tidyverse)
enframe(df) %>%
unnest(value) %>%
group_by(name) %>%
mutate(rn = row_number()) %>%
spread(name, value) %>%
select(-rn)
# A tibble: 4 x 2
# name value
# <dbl> <dbl>
#1 11 -0.484
#2 12 -0.110
#3 13 -0.328
#4 14 0.0737
Or another option is to make use of pivot_longer from the devel version of tidyr
df %>%
set_names(str_c(names(.), "_", cumsum(names(.) == "name"))) %>%
as_tibble %>%
mutate(rn = row_number()) %>%
pivot_longer(-rn, names_to =c(".value", "group"), names_sep = '_') %>%
select(name, value)
Or using base R
reshape(transform(stack(df), rn = ave(seq_along(ind), ind,
FUN = seq_along)), idvar = 'rn', direction = 'wide', timevar = 'ind')
Here's a way in base R using split -
data.frame(
split(v <- unlist(df), sub("[0-9]+", "", names(v)))
)
name value
1 11 -0.2282623
2 12 -0.8101849
3 13 -0.9311898
4 14 0.3638835
Data -
df <- structure(list(name = c(11, 12), value = c(-0.22826229127103,
-0.810184913338659), name = c(13, 14), value = c(-0.931189778412408,
0.363883463286674)), .Names = c("name", "value", "name", "value"
))
d <- data.frame(
name = unlist(df[names(df) == "name"]),
value = unlist(df[names(df) == "value"])
)
the_list <- list(
name=rep(c(11,12), each=1),
value=rnorm(2),
name=rep(c(13,14), each=1),
value=rnorm(2)
)
df <- data.frame(name = unlist(the_list[which(names(the_list) == "name")]),
value = unlist(the_list[which(names(the_list) == "value")]))
df
# name value
# 1 11 -0.83130395
# 2 12 -0.12782566
# 3 13 2.59769395
# 4 14 -0.06967617
Related
I'm trying to process the weather data specified below. I thought I was on the right track but the pivot_longer is not being used in the correct manor and is causing partial duplicates.
Can anyone offer any suggestions as to how I can edit my code? I guess one way would be to perform the pivot_longer after splitting the dataframe into several dataframes i.e. first dataframe - jan, year, second dataframe - feb, year.
maxT <- read.table('https://www.metoffice.gov.uk/pub/data/weather/uk/climate/datasets/Tmax/ranked/England_S.txt', skip = 5, header = TRUE) %>%
select(c(1:24)) %>%
pivot_longer(cols = seq(2,24,2) , values_to = "year") %>%
mutate_at(c(1:12), ~as.numeric(as.character(.))) %>%
pivot_longer(cols = c(1:12), names_to = "month", values_to = "tmax") %>%
mutate(month = match(str_to_title(month), month.abb),
date = as.Date(paste(year, month, 1, sep = "-"), format = "%Y-%m-%d")) %>%
select(-c("name","year","month")) %>%
arrange(date)
Here is an option with tidyverse, using map2
library(dplyr)
library(purrr)
list_df <- maxT %>%
select(seq(1, ncol(.), by = 2)) %>%
map2(maxT %>%
select(seq(2, ncol(.), by = 2)), bind_cols) %>%
imap( ~ .x %>%
rename(!! .y := `...1`, year = `...2`))
-output
map(list_df, head)
#$jan
# A tibble: 6 x 2
# jan year
# <dbl> <int>
#1 9.9 1916
#2 9.8 2007
#3 9.7 1921
#4 9.7 2008
#5 9.5 1990
#6 9.4 1975
#$feb
# A tibble: 6 x 2
# feb year
# <dbl> <int>
#1 11.2 2019
#2 10.7 1998
#3 10.7 1990
#4 10.3 2002
#5 10.3 1945
#6 10 2020
# ...
data
maxT <- read.table('https://www.metoffice.gov.uk/pub/data/weather/uk/climate/datasets/Tmax/ranked/England_S.txt', skip = 5, header = TRUE) %>%
select(c(1:24))
We can use split.default to split group of 2 columns.
list_df <- split.default(maxT, ceiling(seq_along(maxT)/2))
data
maxT <- read.table('https://www.metoffice.gov.uk/pub/data/weather/uk/climate/datasets/Tmax/ranked/England_S.txt', skip = 5, header = TRUE) %>%
select(c(1:24))
Given a (simplified) dataframe with format
df <- data.frame(a = c(1,2,3,4),
b = c(4,3,2,1),
temp1 = c("-","-","-","foo: 3"),
temp2 = c("-","bar: 10","-","bar: 4")
)
a b temp1 temp2
1 4 - -
2 3 - bar: 10
3 2 - -
4 1 foo: 3 bar: 4
I need to rename all temp columns with the names contained within the column, My end goal is to end up with this:
a b foo bar
1 4 - -
2 3 - 10
3 2 - -
4 1 3 4
the df column names and the data contained within them will be unknown, however the columns that need changing will contain temp and the delimiter will always be a ":"
As such I can easily remove the name from within the columns using dplyr like this:
df <- df %>%
mutate_at(vars(contains("temp")), ~(substr(., str_locate(., ":")+1,str_length(.))))
but first I need to rename the columns based on some function method, that scans the column and returns the value(s) within it, ie.
rename_at(vars(contains("temp")), ~(...some function.....))
As per the example given there's no guarantee that specific rows will have data so I can't simply grab value from row 1
Any ideas welcome.
Thanks in advance
One possibility involving dplyr and tidyr could be:
df %>%
pivot_longer(names_to = "variables", values_to = "values", -c(a:b)) %>%
mutate(values = replace(values, values == "-", NA_character_)) %>%
separate(values, into = c("variables2", "values"), sep = ": ") %>%
group_by(variables) %>%
fill(variables2, .direction = "downup") %>%
ungroup() %>%
select(-variables) %>%
pivot_wider(names_from = "variables2", values_from = "values")
a b foo bar
<dbl> <dbl> <chr> <chr>
1 1 4 <NA> <NA>
2 2 3 <NA> 10
3 3 2 <NA> <NA>
4 4 1 3 4
If you want to further replace the NAs with -:
df %>%
pivot_longer(names_to = "variables", values_to = "values", -c(a:b)) %>%
mutate(values = replace(values, values == "-", NA_character_)) %>%
separate(values, into = c("variables2", "values"), sep = ": ") %>%
group_by(variables) %>%
fill(variables2, .direction = "downup") %>%
ungroup() %>%
select(-variables) %>%
pivot_wider(names_from = "variables2", values_from = "values") %>%
mutate_at(vars(-a, -b), ~ replace_na(., "-"))
a b foo bar
<dbl> <dbl> <chr> <chr>
1 1 4 - -
2 2 3 - 10
3 3 2 - -
4 4 1 3 4
This will do the job:
colnames(df)[which(grepl("temp", colnames(df)))] <- unique(unlist(sapply(df[,grepl("temp", colnames(df))],
function(x){gsub("[:].*",
"",
grep("\\w+",
x,
value = TRUE))})))
I have a question:
I have a dataset like this simple example:
df<-data.frame(ID=c("A","B","C","D"),
Score=c("15","16/18/19+2/6","3/+2","19/18/14"))
I want to end up with a dataset that has split the score numbers. I have a problem with the /+2 part. when it says "3/+2"it actually means: "3/3+2" which would finally give "3/5". So what I would like some help with, is to end up with a dataset like this:
ID Score
A 15
B 16/18/19/21/6
C 3/5
D 19/18/14
I already found out that I can then seperate the score by
df<-df %>%
mutate(Score = strsplit(as.character(ID), "/")) %>%
unnest(Score)
But I don't know how I can let the numbers duplicate and then sum when /+ occurs, could someone help me?
It could be probably solved in a more elegant way, but here is one possibility:
df %>%
mutate(Score = strsplit(as.character(Score), "/")) %>%
unnest() %>%
rowwise() %>%
mutate(Score = eval(parse(text = paste0(Score)))) %>%
group_by(ID) %>%
mutate(Score = paste0(Score, collapse = "/")) %>%
distinct()
ID Score
<fct> <chr>
1 A 15
2 B 16/18/21/6
3 C 3/5
4 D 19/18/14
Sample data:
df <- data.frame(ID=c("A","B","C","D"),
Score=c("15","16/18/19+2/6","3/3+2","19/18/14"))
It splits "Score" based on /, converts characters to expression by parse() and then transforms it back.
Using the data you provided and the pattern from #A. Suliman:
df %>%
mutate(Score = strsplit(gsub("(\\d+)/*\\+(\\d+)","\\1/\\1+\\2", Score), "/")) %>%
unnest() %>%
rowwise() %>%
mutate(Score = eval(parse(text = paste0(Score)))) %>%
group_by(ID) %>%
mutate(Score = paste0(Score, collapse = "/")) %>%
distinct()
ID Score
<fct> <chr>
1 A 15
2 B 16/18/19/21/6
3 C 3/5
4 D 19/18/14
We can use gsubfn to do this in a compact way
library(gsubfn)
library(tidyverse)
df %>%
mutate(Score = gsubfn("\\d+\\+\\d+", ~ eval(parse(text = x)), Score))
# ID Score
#1 A 15
#2 B 16/18/21/6
#3 C 3/5
#4 D 19/18/14
data
df <- data.frame(ID=c("A","B","C","D"),
Score=c("15","16/18/19+2/6","3/3+2","19/18/14"), stringsAsFactors = FALSE)
library(dplyr)
library(tidyr) #separate_rows, no need for unnest
df %>% rowwise()%>%
mutate(Score_upd=paste0(sapply(unlist(strsplit(gsub('(\\d+)/*\\+(\\d+)','\\1/\\1+\\2',Score),'/')),
function(x)eval(parse(text = x))),collapse = '/')) %>%
separate_rows(Score_upd,sep = '/')
#short version
df %>% mutate(Score=gsub('(\\d+)/*\\+(\\d+)','\\1/\\1+\\2',Score)) %>%
separate_rows(Score,sep='/') %>% rowwise() %>% mutate(Score=eval(parse(text=Score))) %>%
group_by(ID) %>% summarise(Score=paste0(Score,collapse = '/'))
# A tibble: 4 x 2
ID Score
<fct> <chr>
1 A 15
2 B 16/18/19/21/6
3 C 3/5
4 D 19/18/14
The main idea is using gsub to separate 2+3 correctly, e.g:
gsub('(\\d+)/*\\+(\\d+)','\\1/\\1+\\2','20/8/2+3') #/* means 0 or 1 occurence of / e.g, 19+2 and 3/+2.
[1] "20/8/2/2+3"
Then
valid_str <- gsub('(\\d+)/*\\+(\\d+)','\\1/\\1+\\2','20/8/2+3')
sapply(unlist(strsplit(valid_str,'/')),function(x) eval(parse(text=x)))
20 8 2 2+3
20 8 2 5
#OR
sapply(unlist(strsplit(valid_str,'/')),function(x) sum(as.numeric(unlist(strsplit(x,'\\+')))))
20 8 2 2+3
20 8 2 5
Consider this simple example
> weird_df <- data_frame(col1 =c('hello', 'world', 'again'),
+ col_weird = list(list(12,23), list(23,24), NA))
>
> weird_df
# A tibble: 3 x 2
col1 col_weird
<chr> <list>
1 hello <list [2]>
2 world <list [2]>
3 again <lgl [1]>
I need to extract the values in the col_weird. How can I do that? I see how to do that in Python but not in R. Expected output is:
> good_df
# A tibble: 3 x 3
col1 tic toc
<chr> <dbl> <dbl>
1 hello 12 23
2 world 23 24
3 again NA NA
If you collapse the list column into a string you can use separate from tidyr. I used map from purrr to loop through the list column and create a string with toString.
library(tidyr)
library(purrr)
weird_df %>%
mutate(col_weird = map(col_weird, toString ) ) %>%
separate(col_weird, into = c("tic", "toc"), convert = TRUE)
# A tibble: 3 x 3
col1 tic toc
* <chr> <int> <int>
1 hello 12 23
2 world 23 24
3 again NA NA
You can actually use separate directly without the toString part but you end up with "list" as one of the values.
weird_df %>%
separate(col_weird, into = c("list", "tic", "toc"), convert = TRUE) %>%
select(-list)
This led me to tidyr::extract, which works fine with the right regular expression. If your list column was more complicated, though, writing out the regular expression might be a pain.
weird_df %>%
extract(col_weird, into = c("tic", "toc"), regex = "([[:digit:]]+), ([[:digit:]]+)", convert = TRUE)
You can do this with basic R, thanks to I():
weird_df <- data.frame(col1 =c('hello', 'world'),
col_weird = I(list(list(12,23),list(23,24))))
weird_df
> col1 col_weird
1 hello 12, 23
2 world 23, 24
weird_df <- data_frame(col1 = c('hello', 'world'),
col_weird = list(list(12,23), list(23,24)))
library(dplyr)
weird_df %>%
dplyr::mutate(tic = unlist(magrittr::extract2(col_weird, 1)),
toc = unlist(magrittr::extract2(col_weird, 2)),
col_weird = NULL)
With the last changes: Note that now col_weird contains list(NA, NA)
weird_df <- data_frame(col1 = c('hello', 'world', 'again'),
col_weird = list(list(12,23), list(23,24), list(NA, NA)))
library(dplyr)
weird_df %>%
dplyr::mutate(col_weird = matrix(col_weird),
tic = sapply(col_weird, function(x) magrittr::extract2(x, 1)),
toc = sapply(col_weird, function(x) magrittr::extract2(x, 2)),
col_weird = NULL)
Here is one option to do with purrr/tidyverse/reshape2. We unlist the 'col_weird' within map to get the output as list, set the names of the list with 'col1', melt to 'long' format, grouped by 'L1', create a 'rn' column and spread it back to 'wide'
library(tidyverse)
library(reshape2)
weird_df$col_weird %>%
map(unlist) %>%
setNames(., weird_df$col1) %>%
melt %>%
group_by(L1) %>%
mutate(rn = c('tic', 'toc')[row_number()]) %>%
spread(rn, value) %>%
left_join(weird_df[-2], ., by = c(col1 = "L1"))
well, I came up with a simple one
> weird_df %>%
+ rowwise() %>%
+ mutate(tic = col_weird[[1]],
+ tac = ifelse(length(col_weird) == 2, col_weird[[2]], NA)) %>%
+ select(-col_weird) %>% ungroup()
# A tibble: 3 x 3
col1 tic tac
<chr> <dbl> <dbl>
1 hello 12 23
2 world 23 24
3 again NA NA
I can calculate the rank of the values (val) in my dataframe df within the group name1 with the code:
res <- df %>% arrange(val) %>% group_by(name1) %>% mutate(RANK=row_number())
Instead of writing the column "name1" in the code, I want to pass it as variable, eg crit = "name1". However, the code below does not work since crit1 is assumed to be the column name instead of a variable name.
res <- df %>% arrange(val) %>% group_by(crit1) %>% mutate(RANK=row_number())
How can I pass crit1 in the code?
Thanks.
We can use group_by_
library(dplyr)
df %>%
arrange(val) %>%
group_by_(.dots=crit1) %>%
mutate(RANK=row_number())
#Source: local data frame [10 x 4]
#Groups: name1, name2 [7]
# val name1 name2 RANK
# <dbl> <chr> <chr> <int>
#1 -0.848370044 b c 1
#2 -0.583627199 a a 1
#3 -0.545880758 a a 2
#4 -0.466495124 b b 1
#5 0.002311942 a c 1
#6 0.266021979 c a 1
#7 0.419623149 c b 1
#8 0.444585270 a c 2
#9 0.536585304 b a 1
1#0 0.847460017 a c 3
Update
group_by_ is deprecated in the recent versions (now using dplyr version - 0.8.1), so we can use group_by_at which takes a vector of strings as input variables
df %>%
arrange(val) %>%
group_by_at(crit1) %>%
mutate(RANK=row_number())
Or another option is to convert to symbols (syms from rlang) and evaluate (!!!)
df %>%
arrange(val) %>%
group_by(!!! rlang::syms(crit1)) %>%
mutate(RANK = row_number())
data
set.seed(24)
df <- data.frame(val = rnorm(10), name1= sample(letters[1:3], 10, replace=TRUE),
name2 = sample(letters[1:3], 10, replace=TRUE),
stringsAsFactors=FALSE)
crit1 <- c("name1", "name2")
Update with dplyr 1.0.0
The new across syntax eliminates the need for !!! rlang::syms(). So you can now simplify the code by:
df %>%
arrange(val) %>%
group_by(across(all_of(crit1))) %>%
mutate(RANK = row_number())
Facing a similar task I could successfully work with these two options.
Use across():
for (crit in names(df)) {
print(df |>
# all_of() is not needed here
group_by(across(crit)) |>
count())
}
Use syms() and !!:
crits = syms(names(df))
for (crit in crits) {
print(df |>
# the use of !! instead of !!! is now encouraged
group_by(!!crit) |>
count())
}