Pull out only two variables from a column - r

I have a dataframe in R for which one column has multiple variables. The variables either start with ABC, DEF, GHI. Those variables are followed by a series of 6 numbers (ie ABC052689, ABC062895, DEF045158).
For each row, i would like to pull one instance of ABC (the one with the largest number).
If the row has ABC052689, ABC062895, DEF045158, I would like it to pull out ABC062895 because it is greater than ABC052689.
I would then want to do the same for the variable that starts with DEF######.
I have managed to filter the data to have rows where ABC is there and either DEF or GHI is there:
library(tidyverse)
data_with_ABC <- test %>%
filter(str_detect(car,"ABC"))
data_with_ABC_and_DEF_or_GHI <- data_with_ABC %>%
filter(str_detect(car, "DEF") | str_detect(car, "GHI"))
I don't know how to pull out let's say ABC with the greatest number
ABC052689, ABC062895, DEF045158 -> ABC062895

For a base R solution, we can try using lapply along with strsplit to identify the greatest ABC plate in each CSV string, in each row.
df <- data.frame(car=c("ABC052689,ABC062895,DEF045158"), id=c(1),
stringsAsFactors=FALSE)
df$largest <- lapply(df$car, function(x) {
cars <- strsplit(x, ",", fixed=TRUE)[[1]]
cars <- cars[substr(cars, 1, 3) == "ABC"]
max <- cars[which.max(substr(cars, 4, 9))]
return(max)
})
df
car id largest
1 ABC052689,ABC062895,DEF045158 1 ABC062895
Note that we don't need to worry about casting the substring of the plate number, because it is fixed width text. This means that it should sort properly even as text.

Besides Tim's answer, if you want to do all ABC/DEF at one time, following code may help with library(tidyverse):
> df <- data.frame(car=c("ABC052689", "ABC062895", "DEF045158", "DEF192345"), stringsAsFactors=FALSE)
>
> df2 = df %>%
+ mutate(state = str_sub(car, 1, 3), plate = str_sub(car, 4, 9))
>
> df2
car state plate
1 ABC052689 ABC 052689
2 ABC062895 ABC 062895
3 DEF045158 DEF 045158
4 DEF192345 DEF 192345
>
> df2 %>%
+ group_by(state) %>%
+ summarise(maxplate = max(plate)) %>%
+ mutate(full = str_c(state, maxplate))
# A tibble: 2 x 3
state maxplate full
<chr> <chr> <chr>
1 ABC 062895 ABC062895
2 DEF 192345 DEF192345

Related

If else statement with a value that is part of a continuous character in R

My dataframe (df) contains a list of values which are labelled following a format of 'Month' 'Name of Site' and 'Camera No.'. I.e., if my value is 'DECBUTCAM27' then Dec-December, BUT-Name of Site and CAM27-Camera No.
I have 100 such values with 19 different site names.
I want to write an If else code such that only the site names are recognised and a corresponding number is added.
My initial idea was to add the corresponding number for all the 100 values, but since if else does not work beyond 50 values I couldnt use that option.
This is what I had written for the option that i had tried:
df <- df2 %>% mutate(Site_ID =
ifelse (CT_Name == 'DECBUTCAM27', "1",
ifelse (CT_Name == 'DECBUTCAM28', "1",
ifelse (CT_Name == 'DECI2NCAM01', "2",
ifelse (CT_Name == 'DECI2NCAM07', "2",
ifelse (CT_Name == 'DECI5CAM39', "3",
ifelse (CT_Name == 'DECI5CAM40', "3","NoVal")))))))
I am looking for a code such that only the sites i.e., 'BUT', 'I2N' and 'I5' would be recognised and a corresponding number is added.
Any help would be greatly appreciated.
Extract the sitename using regex and use match + unique to assign unique number.
df2$site_name <- sub('...(.*)CAM.*', '\\1', df2$CT_Name)
df2$Site_ID <- match(df2$site_name, unique(df2$site_name))
For example, see this example :
CT_Name <- c('DECBUTCAM27', 'DECBUTCAM28', 'DECI2NCAM07', 'DECI2NCAM01',
'DECI5CAM39', 'DECI5CAM40')
site_name <- sub('...(.*)CAM.*', '\\1', CT_Name)
site_name
#[1] "BUT" "BUT" "I2N" "I2N" "I5" "I5"
Site_ID <- match(site_name, unique(site_name))
Site_ID
#[1] 1 1 2 2 3 3
Here is a tidyverse solution:
You haven't provided a reproducible example, but let's use the CT_Names that you have supplied to create a test dataframe:
data <- tribble(
~ CT_Name,
"DECBUTCAM27",
"DECBUTCAM28",
"DECI2NCAM01",
"DECI2NCAM07",
"DECI5CAM39",
"DECI5CAM40"
)
Let's assume that the string format is 3 letters for months, 2 or more letters or numbers for site and CAM + 1 or more digits for camera number (adjust these as needed). We can use a regular expression in tidyr's extract() function to split up the string into its components:
data_new <- data %>%
extract(CT_Name, regex = "(\\w{3})(\\w{2,})(CAM\\d+)", into = c("Month", "Site", "Camera"))
(add remove = FALSE if you want to keep the original CT_Name variable)
This yields:
# A tibble: 6 x 3
Month Site Camera
<chr> <chr> <chr>
1 DEC BUT CAM27
2 DEC BUT CAM28
3 DEC I2N CAM01
4 DEC I2N CAM07
5 DEC I5 CAM39
6 DEC I5 CAM40
We can then group by site and assign a group ID as your Site_ID:
data_new <- data %>%
extract(CT_Name, regex = "(\\w{3})(\\w{2,})(CAM\\d+)", into = c("Month", "Site", "Camera")) %>%
group_by(Site) %>%
mutate(Site_ID = cur_group_id())
This produces:
# A tibble: 6 x 4
# Groups: Site [3]
Month Site Camera Site_ID
<chr> <chr> <chr> <int>
1 DEC BUT CAM27 1
2 DEC BUT CAM28 1
3 DEC I2N CAM01 2
4 DEC I2N CAM07 2
5 DEC I5 CAM39 3
6 DEC I5 CAM40 3
Here is a quick example using regex to find the site code and using an apply function to return a vector of code.
df <- data.frame(code = c('DECBUTCAM27','JANBUTCAM27','DECDUCCAM45'))
df$loc <- apply(df, 1, function(x) gsub("CAM.*$","",gsub("^.{3}",'',x[1])))
unique(df$loc) # all the location of the file
df$n <- as.numeric(as.factor(df$loc)) # get a number for each location
Mind that here I use the x[1] because the code are in the first column of my data.frame, which may vary for you.
---EDIT--- This was a previous answer also working but with more work for you to do. However it allow you to choose numeric code value (or text) to assign locations if they are ordered for example.
It require you to put all the codes for each site, which I found heavy in term of code but it works. The switch part is roughly the same as an ifelse.
The regex consist in excluding the 3 first character and the other ones at the end after the 'CAM' sequence.
df <- data.frame(code = c('DECBUTCAM27','JANBUTCAM27','DECDUCCAM45'))
df$n <- apply(df, 1, function(x) switch(gsub("CAM.*$","",gsub("^.{3}",'',x[1])),
BUT = 1,
DUC = 2)
)

remove paired rows where lag == 0 and calculate % change using dplyr and chaining

I'm working with a very large tibble and want to calculate the % of growth of those tables over time (first entry to last entry, not max to min). I would also ultimately want to store any tables with 0 change to their own list/tibble but remove them from the original output table.
an example of the dataset looks like this:
date tbl_name row_cnt
2/12/2019 first 247
6/5/2019 first 247
4/24/2019 second 3617138
6/5/2019 second 3680095
3/1/2019 third 62700321
6/5/2019 third 63509189
4/24/2019 fourth 2
6/5/2019 fourth 2
... ... ...
and the expected output of the table would be two tables that would appear as such:
tbl_name pct_change
second 1.74
third 1.29
... ...
tbl_name
first
fourth
...
so far I have been able to arrange the observations, group them, and filter the first and last instance of each group successfully with:
test_df <- df %>%
arrange(l.fully_qualf_tbl_nm) %>%
group_by(l.fully_qualf_tbl_nm) %>%
filter(row_number()==1 | row_number()==n()) %>%
mutate(pct_change = ((l.row_cnt/lag(l.row_cnt) - 1) * 100)) %>%
select(l.run_dt, l.fully_qualf_tbl_nm, pct_change) %>%
drop_na(pct_change)
but my calculation
mutate(pct_change = ((l.row_cnt/lag(l.row_cnt) - 1) * 100)) %>%
is not generating the right results. I pulled my pct-change calculation from another SO post that discusses %-change but i'm getting different numbers from my hand-calculations.
For example, I'm getting "second = 3.61" but a hand-calculation (as well as excel) gets 1.74. I'm also getting "third = 0.831" instead of 1.29 by-hand. My guess is that I'm not properly specifying that I only want the calculation done on each group (each pair of two rows). I'm wondering if I should be calculating lag separately or if I am just implementing lag() incorrectly?
next, I think the new table would be created with some manner of
if return value of filter(row_number()==1 | row_number()==n()) %>% == 0, append to list/table
but I honestly, have no clue how to do this. I'm wondering if I should just do a separate function and assign it to a new variable.
df <- read.table(
header = T,
stringsAsFactors = F,
text = " date tbl_name row_cnt
2/12/2019 first 247
6/5/2019 first 247
4/24/2019 second 3617138
6/5/2019 second 3680095
3/1/2019 third 62700321
6/5/2019 third 63509189
4/24/2019 fourth 2
6/5/2019 fourth 2")
# Wrapping in parentheses assigns the output to test_df and also prints it
(test_df <- df %>%
group_by(tbl_name) %>%
mutate(pct_change = ((row_cnt/lag(row_cnt) - 1) * 100)) %>%
ungroup() %>%
filter(!is.na(pct_change)) %>% # Filter after pct_change calc, since we want to
# include change from 1:2 and from n-1:n
select(tbl_name, row_cnt, pct_change))
# A tibble: 4 x 3
tbl_name row_cnt pct_change
<chr> <int> <dbl>
1 first 247 0
2 second 3680095 1.74
3 third 63509189 1.29
4 fourth 2 0
To split into two tables, it seems one could do:
first_tbl <- test_df %>% filter(pct_change != 0) # or "pct_change > 0" for pos growth
second_tbl <- test_df %>% filter(pct_change == 0)

Add also the empty rows in a data frame construction

I want to construct a data frame based on two data frames
Here it is an example
#toy example
name <- c("Li", "Pedro", "Dave")
age <- c(20, 30, 40)
d1 <- cbind.data.frame(name, age)
name <- c("Pedro", "Dave", "Grace")
fav_col <- c("red", "blue", "yellow")
lastname <- c("Sanchez", "Stone", "Flint")
fav_food <- c("pizza", "hamburguers", "salad")
d2 <- cbind.data.frame(name, fav_col, lastname, fav_food)
d1$name <- as.character(d1$name)
d2$name <- as.character(d2$name)
cols <- c()
for(i in 1:nrow(d1)) {
some <- dplyr::filter(d2, name==d1$name[i])
cols <- rbind.data.frame(cols, data.frame(some$name, some$fav_col, some$fav_food))
}
Doing this I am obtaining a data frame called "cols" and looks like this:
some.name some.fav_col some.fav_food
Pedro red pizza
Dave blue hamburguers
But what I want is
some.name some.fav_col some.fav_food
NA(or empty) NA(or empty) Na(or empty)
Pedro red pizza
Dave blue hamburguers
The first iteration when i = 1 must produce an empty exit because there is no Li in the second data frame, and I want this empty space in my data frame. Do you know how I could get this?
At the end I want to add the second and third columns of "cols" to "d1" to get:
name age fav_col fav_food
Li 20 NA (or empty) NA (or empty)
Pedro 30 red pizza
Dave 40 blue hamburguers
Also I don't want the empty spaces that the second data frame could produce like this:
name age fav_col fav_food
Li 20 NA NA
Pedro 30 red pizza
Dave 40 blue hamburguers
Grace NA yellow salad
I just want to merge the tables keeping only the names of the first data frame and add the two extra columns. I would appreciate any help
You can use union_all from dplyr.
library(tidyr)
library(dplyr)
df <- union_all(d1, d2) %>%
mutate_if(is.factor, as.character) %>% # only required when your text columns are
group_by(name) %>% # identified as factor and not character.
summarise_all(max, na.rm = TRUE) %>% # Because max only works on numeric or char
ungroup
df
# # A tibble: 3 x 4
# name age fav_col lastname
# <chr> <dbl> <chr> <chr>
# 1 Dave 40. blue Stone
# 2 Li 20. NA NA
# 3 Pedro 30. red Sanchez
To get your desired output, you can add drop_na to the chain.
df %>% select(name, fav_col) %>% drop_na
# # A tibble: 2 x 2
# name fav_col
# <chr> <chr>
# 1 Dave blue
# 2 Pedro red
I used this, and I added paste and collapse just in case that someone needs to add results from different cells in just one.
f_add_col <- function(vec) {
add_col <- dplyr::filter(d2, name==vec[1])
return (paste(add_col$fav_col, collapse = "|"))
}
cbind.data.frame(d1, fav_col=apply(d1, 1, f_add_col))
Then I did the same for the column fav_food.

Subset rows for each group based on a character in a column and order of occurrence in a data frame

I have a data similar to this.
B <- data.frame(State = c(rep("Arizona", 8), rep("California", 8), rep("Texas", 8)),
Account = rep(c("Balance", "Balance", "In the Bimester", "In the Bimester", "Expenses",
"Expenses", "In the Bimester", "In the Bimester"), 3), Value = runif(24))
You can see that Account has 4 occurrences of the element "In the Bimester", two "chunks" of two elements for each state, "Expenses" in between them.
The order here matters because the first chunk is not referring to the same thing as the second chunk.
My data is actually more complex, It has a 4th variable, indicating what each row of Account means. The number of its elements for each Account element (factor per se) can change. For example, In some state, the first "chunk" of "In the Bimester" can have 6 rows and the second, 7; but, I cannot differentiate by this 4th variable.
Desired: I'd like to subset my data, spliting those two "In the Bimester" by each state, subsetting only the rows of the first "chunks" by each state or the second "chunks".
I have a solution using data.table package, but I'm finding it kind of poor. any thoughts?
library(data.table)
B <- as.data.table(B)
B <- B[, .(Account, Value, index = 1:.N), by = .(State)]
x <- B[Account == "Expenses", .(min_ind = min(index)), by = .(State)]
B <- merge(B, x, by = "State")
B <- B[index < min_ind & Account == "In the Bimester", .(Value), by = .(State)]
You can use dplyr package:
library(dplyr)
B %>% mutate(helper = data.table::rleid(Account)) %>%
filter(Account == "In the Bimester") %>%
group_by(State) %>% filter(helper == min(helper)) %>% select(-helper)
# # A tibble: 6 x 3
# # Groups: State [3]
# State Account Value
# <fctr> <fctr> <dbl>
# 1 Arizona In the Bimester 0.17730148
# 2 Arizona In the Bimester 0.05695585
# 3 California In the Bimester 0.29089678
# 4 California In the Bimester 0.86952723
# 5 Texas In the Bimester 0.54076144
# 6 Texas In the Bimester 0.59168138
If instead of min you use max you'll get the last occurrences of "In the Bimester" for each State. You can also exclude Account column by changing the last pipe to select(-helper,-Account).
p.s. If you don't want to use rleid from data.table and just use dplyr functions take a look at this thread.

Reshape data frame for consecutive years

I have data about thousands of customers who visited stores in the 3 past years.
For each customer, I have :
ID
Combination of a year and the first store visited in this year.
Customer_Id | Year_*_Store
1 2010_A
1 2011_B
1 2012_C
2 2010_A
2 2011_B
2 2012_D
What I’d like to have is the following structure of data in order to visualize the evolution of the customers’behaviour with a riverplot( aka Sankey plot)
For instance the 2 customers, who firstly visited the store A in 2010, firstly visited the store B in 2011:
SOURCE | TARGET | NB_CUSTOMERS
2010_A 2011_B 2
2011_B 2012_C 1
2011_B 2012_D 1
I don't want links between two years which are not consecutive like 2010_A and 2012_D
How can I do that in R ?
I would do this with dplyr (faster)
df<-read.table(header=T,text="Customer_Id Year_Store
1 2010_A
1 2011_B
1 2012_C
2 2010_A
2 2011_B
2 2012_D")
require(dplyr) # for aggregation
require(riverplot) # for Sankey
targets<-
group_by(df,Customer_Id) %.% # group by Customer
mutate(source=Year_Store,target=c(as.character(Year_Store)[-1],NA)) %.% # add a lag to show the shift
filter(!is.na(target)) %.% # filter out empty edges
regroup(list("source","target")) %.% # regroup by source & target
summarise(len=length(Customer_Id)) %.% # count customers for relationship
mutate(step=as.integer(substr(target,1,4))-as.integer(substr(source,1,4))) %.% # add a step to show how many years
filter(step==1) # filter out relationships for non consec years
topnodes <- c(as.character(unique(df$Year_Store))) # unique nodes
nodes <- data.frame( ID=topnodes, # IDs
x=as.numeric(substr(topnodes,1,4)), # x value for plot
col= rainbow(length(topnodes)), # color each different
labels= topnodes, # labels
stringsAsFactors= FALSE )
edges<- # create list of list
lapply(unique(targets$source),function(x){
l<-as.list(filter(targets,source==x)$len) # targets per source
names(l)<-filter(targets,source==x)$target # name of target
l
})
names(edges)<-unique(targets$source) # name top level nodes
r <- makeRiver( nodes, edges) # make the River
plot( r ) # plot it!
Note that you can't have a * in column names (see ?make.names). Here is a basic approach:
Split Year_store into two separate columns Year and Store in your data frame; at the moment it contains two completely different kinds of data and you actually need to process them separately.
Make a NextYear column, defined as Year + 1
Make a NextStore column, in which you assign the store code matching Customer_Id and for which Year is the same as this row's NextYear, assigning NA if there is no record of the customer visiting a store the next year, and throwing an error if the data do not meet the required specification (are ambiguous about which store was visited first the next year).
Strip out any of the rows in which NextStore is NA, and combine the NextYear and NextStore columns into a NextYear_NextStore column.
Summarize your data frame by the Year_store and NextYear_NextStore columns e.g. using ddply in the plyr package.
Some sample data:
# same example data as question
customer.df <- data.frame(Customer_Id = c(1, 1, 1, 2, 2, 2),
Year_Store = c("2010_A", "2011_B", "2012_C", "2010_A", "2011_B", "2012_D"),
stringsAsFactors = FALSE)
# alternative data should throw error, customer 2 is inconsistent in 2011
badCustomer.df <- data.frame(Customer_Id = c(1, 1, 1, 2, 2, 2),
Year_Store = c("2010_A", "2011_B", "2012_C", "2010_A", "2011_B", "2011_D"),
stringsAsFactors = FALSE)
And an implementation:
require(plyr)
splitYearStore <- function(df) {
df$Year <- as.numeric(substring(df$Year_Store, 1, 4))
df$Store <- as.character(substring(df$Year_Store, 6))
return(df)
}
findNextStore <- function(df, matchCust, matchYear) {
matchingStore <- with(df,
df[Customer_Id == matchCust & Year == matchYear, "Store"])
if (length(matchingStore) == 0) {
return(NA)
} else if (length(matchingStore) > 1) {
errorString <- paste("Inconsistent store results for customer",
matchCust, "in year", matchYear)
stop(errorString)
} else {
return(matchingStore)
}
}
tabulateTransitions <- function(df) {
df <- splitYearStore(df)
df$NextYear <- df$Year + 1
df$NextStore <- mapply(findNextStore, matchCust = df$Customer_Id,
matchYear = df$NextYear, MoreArgs = list(df = df))
df$NextYear_NextStore <- with(df, paste(NextYear, NextStore, sep = "_"))
df <- df[!is.na(df$NextStore),]
df <- ddply(df, .(Source = Year_Store, Target = NextYear_NextStore),
summarise, No_Customers = length(Customer_Id))
return(df)
}
Results:
> tabulateTransitions(customer.df)
Source Target No_Customers
1 2010_A 2011_B 2
2 2011_B 2012_C 1
3 2011_B 2012_D 1
> tabulateTransitions(badCustomer.df)
Error in function (df, matchCust, matchYear) :
Inconsistent store results for customer 2 in year 2011
No attempt has been made to optimise; if your data set is massive then perhaps you should investigate a data.table solution.

Resources