I have a dataframe with the following factor variable:
> head(example.df)
path
1 C:/Users/My PC/pinkhipppos/tinyhorsefeet/location1/categoryA/eyoshdzjow_random_image.txt
(made up dirs).
I want to split into separate columns based on a delimiter: /.
I can do this using
library(tidyverse)
example.df <- example.df %>%
separate(path,
into=c("dir",
"ok",
"hello",
"etc...",
"finally...",
"location",
"category",
"filename"),
sep="/")
Although, I am only interested in the last two dirs and the file name or the last 3 results from the separate function. As parent directories (higher than location) may change. My desired output would be:
> head(example.df)
location category filename
1 location1 categoryA eyoshdzjow_random_image.txt
Reproducible:
example.df <- as.data.frame(
c("C:/Users/My PC/pinkhipppos/tinyhorsefeet/location1/categoryA/eyoshdzjow_random_image.txt",
"C:/Users/My PC/pinkhipppos/tinyhorsefeet/location2/categoryB/jdugnbtudg_random_image.txt")
)
colnames(example.df)<-"path"
One way in base R is to split string at "/" and select last 3 elements from each list.
as.data.frame(t(sapply(strsplit(as.character(example.df$path), "/"), tail, 3)))
# V1 V2 V3
#1 location1 categoryA eyoshdzjow_random_image.txt
#2 location2 categoryB jdugnbtudg_random_image.txt
Using tidyverse, we can get the data in long format, select last 3 entries in each row and get the data in wide format.
library(tidyverse)
example.df %>%
mutate(row = row_number()) %>%
separate_rows(path, sep = "/") %>%
group_by(row) %>%
slice((n() - 2) : n()) %>%
mutate(cols = c('location', 'category', 'filename')) %>%
pivot_wider(names_from = cols, values_from = path) %>%
ungroup() %>%
select(-row)
# A tibble: 2 x 3
# location category filename
# <chr> <chr> <chr>
#1 location1 categoryA eyoshdzjow_random_image.txt
#2 location2 categoryB jdugnbtudg_random_image.txt
Or similar concept as base R but using tidyverse
example.df %>%
mutate(temp = map(str_split(path, "/"), tail, 3)) %>%
unnest_wider(temp, names_repair = ~paste0("dir", seq_along(.) - 1)) %>%
select(-dir0)
Related
Context
I have created a small sample dataframe to explain my problem. The original one is larger, as it has many more columns. But it is formatted in the same way.
df = data.frame(Case1.1.jpeg.text="the",
Case1.1.jpeg.text.1="big",
Case1.1.jpeg.text.2="DOG",
Case1.1.jpeg.text.3="10197",
Case1.2.png.text="framework",
Case1.3.jpg.text="BE",
Case1.3.jpg.text.1="THE",
Case1.3.jpg.text.2="Change",
Case1.3.jpg.text.3="YOUWANTTO",
Case1.3.jpg.text.4="SEE",
Case1.3.jpg.text.5="in",
Case1.3.jpg.text.6="theWORLD",
Case1.4.png.text="09.80.56.60.77")
The dataframe consists of output from a text detection ML model based on a certain number of input images.
The output format makes each word for each image a separate column, thereby creating a very wide dataset.
Desired Output
I am looking to create a cleaner version of it, with one column containing the image name (e.g. Case1.2.png) and the second with the concatenation of all possible words that the model finds in that particular image (the number of words varies from image to image).
result = data.frame(Case=c('Case1.1.jpeg','Case1.2.png','Case1.3.jpg','Case1.4.png'),
Text=c('thebigDOG10197','framework','BETHEChangeYOUWANTTOSEEintheWORLD','09.80.56.60.77'))
I have tried many approaches based on similar questions found on Stackoverflow, but none seem to give me the exact output I'm looking for.
Any help on this would be greatly appreciated.
library(tidyr)
library(dplyr)
df %>%
pivot_longer(cols = everything(),
names_pattern = "(.*)\\.(text.*)",
names_to = c("Case", NA)) %>%
group_by(Case) %>%
summarize(value = paste(value, collapse = ""), .groups = "drop")
Alternatively, this can be accomplished using just the pivot functions from tidyr:
library(tidyr)
library(stringr)
df %>%
pivot_longer(cols = everything(),
names_pattern = "(.*)\\.(text).*",
names_to = c("Case", "cols")) %>%
pivot_wider(id_cols = Case,
values_from = value,
names_from = cols,
values_fn = str_flatten)
Output
Case value
<chr> <chr>
1 Case1.1.jpeg thebigDOG10197
2 Case1.2.png framework
3 Case1.3.jpg BETHEChangeYOUWANTTOSEEintheWORLD
4 Case1.4.png 09.80.56.60.77
A possible solution:
library(tidyverse)
df %>%
pivot_longer(everything()) %>%
mutate(name = str_remove(name, "\\.text\\.*\\d*")) %>%
group_by(name) %>%
summarise(text = str_c(value, collapse = ""))
#> # A tibble: 4 x 2
#> name text
#> <chr> <chr>
#> 1 Case1.1.jpeg thebigDOG10197
#> 2 Case1.2.png framework
#> 3 Case1.3.jpg BETHEChangeYOUWANTTOSEEintheWORLD
#> 4 Case1.4.png 09.80.56.60.77
An option in base R is stack the data into a two column data.frame with stack and then do a group by paste with aggregate
aggregate(cbind(Text = values) ~ Case, transform(stack(df),
Case = trimws(ind, whitespace = "\\.text.*")), FUN = paste, collapse = "")
Case Text
1 Case1.1.jpeg thebigDOG10197
2 Case1.2.png framework
3 Case1.3.jpg BETHEChangeYOUWANTTOSEEintheWORLD
4 Case1.4.png 09.80.56.60.77
You can use pivot_longer(everything()), manipulate the "Case" column, group, and paste together:
pivot_longer(df,everything(),names_to="Case") %>%
mutate(Case = str_remove_all(Case, ".text.*")) %>%
group_by(Case) %>% summarize(Text=paste(value, collapse=""))
Output:
Case Text
<chr> <chr>
1 Case1.1.jpeg thebigDOG10197
2 Case1.2.png framework
3 Case1.3.jpg BETHEChangeYOUWANTTOSEEintheWORLD
4 Case1.4.png 09.80.56.60.77
I try to find the most frequent category within every row of a dataframe. A category can consist of multiple words split by a /.
library(tidyverse)
library(DescTools)
# example data
id <- c(1, 2, 3, 4)
categories <- c("apple,shoes/socks,trousers/jeans,chocolate",
"apple,NA,apple,chocolate",
"shoes/socks,NA,NA,NA",
"apple,apple,chocolate,chocolate")
df <- data.frame(id, categories)
# the solution I would like to achieve
solution <- df %>%
mutate(winner = c("apple", "apple", "shoes/socks", "apple"),
winner_count = c(1, 2, 1, 2))
Based on these answers I have tried the following:
Write a function that finds the most common word in a string of text using R
trial <- df %>%
rowwise() %>%
mutate(winner = names(which.max(table(categories %>% str_split(",")))),
winner_count = which.max(table(categories %>% str_split(",")))[[1]])
Also tried to follow this approach, however it also does not give me the required results
How to find the most repeated word in a vector with R
trial2 <- df %>%
mutate(winner = DescTools::Mode(str_split(categories, ","), na.rm = T))
I am mainly struggling because my most frequent category is not just one word but something like "shoes/socks" and the fact that I also have NAs. I don't want the NAs to be the "winner".
I don't care too much about the ties right now. I already have a follow up process in place where I handle the cases that have winner_count = 2.
split the categories on comma in separate rows, count their occurrence for each id, drop the NA values and select the top occurring row for each id
library(dplyr)
library(tidyr)
df %>%
separate_rows(categories, sep = ',') %>%
count(id, categories, name = 'winner_count') %>%
filter(categories != 'NA') %>%
group_by(id) %>%
slice_max(winner_count, n = 1, with_ties = FALSE) %>%
ungroup %>%
rename(winner = categories) %>%
left_join(df, by = 'id') -> result
result
# id winner winner_count categories
# <dbl> <chr> <int> <chr>
#1 1 apple 1 apple,shoes/socks,trousers/jeans,chocolate
#2 2 apple 2 apple,NA,apple,chocolate
#3 3 shoes/socks 1 shoes/socks,NA,NA,NA
#4 4 apple 2 apple,apple,chocolate,chocolate
So I have a column with values in this structure:
tribble(
~col,
"AA_BB;AA_AA;AA_BB",
"BB_BB;AA_AA",
"AA_BB",
"BB_AA;BB_AA;AA_AA;BB_AA")
)
So each row has items separated by a ";". The first for has items AA_BB, AA_AA and AA_BB. I want the first row to be transformed to "AA_BB;AA_AA" and the last row to be transformed to "BB_AA;AA_AA".
I thought about using separate but I the result didn't really help me (especially since I don't know how many columns there can be at most).
df %>%
separate(col, into = c("A", "B", "C", "D"), sep = ";")
Any tips on how to do this?
We can split the column, get the unique elements and paste
library(dplyr)
library(stringr)
library(purrr)
df %>%
mutate(col = map_chr(strsplit(col, ";"), ~ str_c(unique(.x), collapse=";")))
-output
# A tibble: 4 x 1
# col
# <chr>
#1 AA_BB;AA_AA
#2 BB_BB;AA_AA
#3 AA_BB
#4 BB_AA;AA_AA
Or split with separate_rows, then do a group by paste after getting the distinct rows
library(tidyr)
df %>%
mutate(rn = row_number()) %>%
separate_rows(col, sep=";") %>%
distinct %>%
group_by(rn) %>%
summarise(col = str_c(col, collapse=";"), .groups = 'drop') %>%
select(col)
In base R, you can split the string on semi-colon, keep only unique strings and paste them together.
df$col1 <- sapply(strsplit(df$col, ';'), function(x)
paste0(unique(x), collapse = ';'))
df
# A tibble: 4 x 2
# col col1
# <chr> <chr>
#1 AA_BB;AA_AA;AA_BB AA_BB;AA_AA
#2 BB_BB;AA_AA BB_BB;AA_AA
#3 AA_BB AA_BB
#4 BB_AA;BB_AA;AA_AA;BB_AA BB_AA;AA_AA
I have strings containing enumerations of words grouped under word type. The example below only has one type for simplicity's sake.
ka = tibble(
words = c('apple, orange', 'pear, apple, plum'),
type = 'fruit'
)
I want to find out the number of UNIQUE words per type.
I figured I would split the character vectors,
ka = ka %>%
mutate(
word_list = str_split(words, ', ')
)
and then bind the columns per group. The end result would be
c(
ka$word_list[[1]],
ka$word_list[[2]],
)
Then I can unique these vectors and get their length.
I don't know how to bind columns together, grouped by a separate column. I could do this with an ugly loop within a loop, but there must be a map/apply solution as well, following the logic of:
ka %>%
group_by(type) %>%
summarise(
biglist = map(word_list, ~ c(.)), # this doesn't work, obviously
biglist_unique = map(biglist, ~ unique(.)),
biglist_length = map(biglist_unique, ~ length(.))
)
Here is an option for you. First we collapse the vectors, then we map out what you're looking for. Note that we have to trim off the whitespace to get the proper unique words.
library(tidyverse)
ka %>%
group_by(type) %>%
summarise(all_words = paste(words, collapse = ",")) %>%
mutate(biglist = str_split(all_words, ",") %>% map(., ~str_trim(.x, "both")),
biglist_unique = map(biglist, ~.x[unique(.x)]),
biglist_length = map_dbl(biglist_unique, length))
#> # A tibble: 1 x 5
#> type all_words biglist biglist_unique biglist_length
#> <chr> <chr> <list> <list> <dbl>
#> 1 fruit apple, orange,pear, apple, plum <chr [5]> <chr [4]> 4
Another option would be to use tidy data principles and the tidyr package.
ka = ka %>%
mutate(
word_list = str_split(words, ', ')
)
ka %>%
# If you need to maintain information about each row you can create an index
# mutate(index = row_number()) %>%
# unnest the wordlist to get one word per row
unnest(word_list) %>%
# Only keep unique words per group
group_by(type) %>%
distinct(word_list, .keep_all = FALSE) %>% # if you need to maintain row info .keep_all = TRUE
summarise(n_unique = n())
# A tibble: 1 x 2
# type n_unique
# <chr> <int>
# 1 fruit 4
Here's a way you can do using separate_rows:
ka %>%
separate_rows(words, sep = ', ') %>%
group_by(type) %>%
summarise(word_c = n_distinct(words))
Something like this:
library(tidyverse)
ka %>%
mutate(words = strsplit(as.character(words), ",")) %>%
unnest(words) %>%
mutate(words = gsub(" ","",words)) %>%
group_by(type) %>%
summarise(number = n_distinct(words),
words = paste0(unique(words), collapse =' '))
# A tibble: 1 x 3
type number words
<chr> <int> <chr>
1 fruit 4 apple orange pear plum
I am trying to split a column in a data set that has codes separated by "-". This creates two issues. First i have to split the columns, but I also want to impute the values implied by the "-". I was able to split the data using:
separate_rows(df, code, sep = "-")
but I still haven't found a way to impute the implied values.
name <- c('group1', 'group1','group1','group2', 'group1', 'group1',
'group1')
code <- c('93790', '98960 - 98962', '98966 - 98969', '99078', 'S5950',
'99241 - 99245', '99247')
df <- data.frame( name, code)
what I am trying to output would look something like:
group1 93790, 98960, 98961, 98962, 98966, 98967, 98968, 98969, S5950, 99241,
99242, 99243, 99244, 99245, 99247
group2 99078
in this example, 98961, 98967 and 98968 are imputed and implied from the "-".
Any thoughts on how to accomplish this?
After we split the 'code', one option it to loop through the split elements with map, get a sequence (:), unnest and do a group_by paste
library(dplyr)
library(stringr)
library(tidyr)
library(purrr)
df %>%
mutate(code = map(strsplit(as.character(code), " - "), ~ {
x <- as.numeric(.x)
if(length(x) > 1) x[1]:x[2] else x})) %>%
unnest(code) %>%
group_by(name) %>%
summarise(code = str_c(code, collapse=", "))
# A tibble: 2 x 2
# name code
# <fct> <chr>
# 1 group1 93790, 98960, 98961, 98962, 98966, 98967, 98968, 98969
# 2 group2 99078
Or another option is before the separate_rows, create a row index and use that for grouping by when we do a complete
df %>%
mutate(rn = row_number()) %>%
separate_rows(code, convert = TRUE) %>%
group_by(rn, name) %>%
complete(code = min(code):max(code)) %>%
group_by(name) %>%
summarise(code = str_c(code, collapse =", "))
Update
If there are non-numeric elements
df %>%
mutate(rn = row_number()) %>%
separate_rows(code, convert = TRUE) %>%
group_by(name, rn) %>%
complete(code = if(any(str_detect(code, '\\D'))) code else
as.character(min(as.numeric(code)):max(as.numeric(code)))) %>%
group_by(name) %>%
summarise(code = str_c(code, collapse =", "))
# A tibble: 2 x 2
# name code
# <fct> <chr>
#1 group1 93790, 98960, 98961, 98962, 98966, 98967, 98968, 98969, S5950, 99241, 99242, 99243, 99244, 99245, 99247
#2 group2 99078
lapply(split(as.character(df$code), df$name), function(y) {
unlist(sapply(y, function(x){
if(grepl("-", x)) {
n = as.numeric(unlist(strsplit(x, "-")))
n[1]:n[2]
} else {
as.numeric(x)
}
}, USE.NAMES = FALSE))
})
#$group1
#[1] 93790 98960 98961 98962 98966 98967 98968 98969
#$group2
#[1] 99078