web scraping a table of mutations from the COSMIC database - web-scraping

Question

I am having difficulty scraping a table of mutations in the MYC gene from the COSMIC database using rvest. I only get an empty list. The output is below. I have confirmed that the values are in the HTML file (i.e., not JAVA as the values are in the HTML file itself). I have also confirmed that the element I am trying to scrape is a table. I have tried using xpath and CSS. I have also confirmed permission to scrape. Please advise
R Console Output
> library("rvest")
> library("dplyr")
> library("robotstxt")
> library("XML")
> library("RSelenium")
> library("splashr")
> library("reticulate")
> url = "https://cancer.sanger.ac.uk/cosmic/gene/analysis?ln=MYC#variants"
> paths_allowed(url)
cancer.sanger.ac.uk No encoding supplied: defaulting to UTF-8.
[1] TRUE
> Xpath = "//*[#id= 'DataTables_Table_0']"
> a = read_html(url) %>% html_nodes(xpath = Xpath) %>% html_table()
> a
list()
> Selector = "#DataTables_Table_0"
> a = read_html(url) %>% html_nodes(css = Selector) %>% html_table()
> a
list()

Answer 1

You can do the same request the webpage does to retrieve results dynamically (viewable in network tab of dev tools F12.) Alter the DisplayLength param to be all results (1394) or set it to an initial high number and inspect the return to capture the actual total result count and issue any further requests necessary to get all results.
Whilst you can do a simple rvest request
library(rvest)
url <- 'https://cancer.sanger.ac.uk/cosmic/gene/mutations?all_data=&coords=AA%3AAA&dr=&end=455&gd=&id=359910&ln=MYC&seqlen=455&src=gene&start=1&export=json&sEcho=2&iColumns=6&sColumns=&iDisplayStart=0&iDisplayLength=1394&mDataProp_0=0&sSearch_0=&bRegex_0=false&bSearchable_0=true&bSortable_0=true&mDataProp_1=1&sSearch_1=&bRegex_1=false&bSearchable_1=true&bSortable_1=true&mDataProp_2=2&sSearch_2=&bRegex_2=false&bSearchable_2=true&bSortable_2=true&mDataProp_3=3&sSearch_3=&bRegex_3=false&bSearchable_3=true&bSortable_3=true&mDataProp_4=4&sSearch_4=&bRegex_4=false&bSearchable_4=true&bSortable_4=true&mDataProp_5=5&sSearch_5=&bRegex_5=false&bSearchable_5=true&bSortable_5=true&sSearch=&bRegex=false&iSortCol_0=0&sSortDir_0=asc&iSortingCols=1'
r <- read_html(url) %>% html_node('p') %>% html_text()
write.table(r,file="data.txt", sep='\t', row.names = FALSE)
EDIT to the above by #Snehal Patel to get desired format:
x = read.table("data.txt", sep = "\t", skip = 2, fill = TRUE)
colnames(x) = c("AA_Position", "CDS_Mutation", "AA_Mutation", "COSMIC_ID", "count", "Mutation_type")
With httr, passing various headers, and constructing a dataframe from the response.
library(httr)
library(purrr)
library(rvest)
headers = c(
'X-Requested-With' = 'XMLHttpRequest',
'User-Agent' = 'Mozilla/5.0',
'Referer' = 'https://cancer.sanger.ac.uk/cosmic/gene/analysis?ln=MYC'
)
params = list(
'coords' = 'AA:AA',
'end' = '455',
'id' = '359910',
'ln' = 'MYC',
'seqlen' = '455',
'src' = 'gene',
'start' = '1',
'export' = 'json',
'sEcho' = '4',
'iColumns' = '6',
'iDisplayStart' = '0',
'iDisplayLength' = '1394', #for all results. You can set to number higher than you expect then check first result for actual
'mDataProp_0' = '0',
'bRegex_0' = 'false',
'bSearchable_0' = 'true',
'bSortable_0' = 'true',
'mDataProp_1' = '1',
'bRegex_1' = 'false',
'bSearchable_1' = 'true',
'bSortable_1' = 'true',
'mDataProp_2' = '2',
'bRegex_2' = 'false',
'bSearchable_2' = 'true',
'bSortable_2' = 'true',
'mDataProp_3' = '3',
'bRegex_3' = 'false',
'bSearchable_3' = 'true',
'bSortable_3' = 'true',
'mDataProp_4' = '4',
'bRegex_4' = 'false',
'bSearchable_4' = 'true',
'bSortable_4' = 'true',
'mDataProp_5' = '5',
'bRegex_5' = 'false',
'bSearchable_5' = 'true',
'bSortable_5' = 'true',
'bRegex' = 'false',
'iSortCol_0' = '0',
'sSortDir_0' = 'asc',
'iSortingCols' = '1'
)
r <- content(httr::GET(url = 'https://cancer.sanger.ac.uk/cosmic/gene/mutations', httr::add_headers(.headers=headers), query = params)) %>%
.$aaData
df <- map_df(r, function(i) {
data.frame(
`Position` = read_html(i[[1]]) %>% html_node('a') %>% html_text() %>% as.numeric() ,
`CDS Mutation` = read_html(i[[2]]) %>% html_node('a') %>% html_text(),
`AA Mutation` = read_html(i[[3]]) %>% html_node('a') %>% html_text(),
`Legacy Mutation ID` = i[[4]],
`Count` = i[[5]] ,
`Type` = i[[6]] ,
stringsAsFactors=FALSE)
})