quirky behavoir with rvest wikipedia scraping - r

Question

Sorry, this is a weird question, but I just can't seem to figure it out myself. The good news is: I think it's totally reproducible.
I'm trying to build a simple R function to use {rvest} to scrape wikipedia for the hometown of musicians. Basically, the function I wrote works, but for certain artists, it doesn't work (returns NULL). (Randy Newman is one such, so I'll use him as an example.)
When I just run the whole thing (below) and then findHome("randy newman") I get NULL but then when I attempt to debug, I run the tableMusic() function and then artist <- "randy newman" and then run all the guts of the artistData() function line by line, IT WORKS!
AND THEN, once I've done that, I can run findHome("randy newman") and it will work right. What gives?! Do I have something in the wrong order or something? I can't seem to figure it out.
Any help is much appreciated. Here is the code:
library(rvest)
findHome <- function(artist) {
##function to look for the table with the right info
tableMusic <- function(data) {
if(!any(grepl("years active|labels|instruments", data[,1], ignore.case=T))) {
for (i in 2:5) {
data <- try(url %>% html %>% html_nodes(xpath=paste('//*[#id="mw-content-text"]/table[', i, ']', sep="")) %>% html_table(fill=T), silent=T)
if(!class(data)=="try-error" & length(data)>0) {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
if(any(grepl("years active|labels|instruments", data[,1], ignore.case=T))) {
break
}
}
}
}
if(class(data)=="try-error" | length(data)<1) {
data <- NULL
} else if (!any(grepl("years active|labels|instruments", data[,1], ignore.case=T))) {
data <- NULL
}
data
}
#function to pull data and try different pages if the first is wrong
artistData <- function(artist) {
artist <- gsub(" ", "_", artist)
artist <- gsub("'", "%27", artist)
##first try getting the data
url <- paste("https://en.wikipedia.org/wiki/", artist, sep="")
data <- try(url %>% html %>% html_nodes(xpath='//*[#id="mw-content-text"]/table[1]') %>% html_table(fill=T), silent=T)
##check if it's the right page (deal with disambiguation issues)
if(!class(data)=="try-error" & length(data)>0) {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
data <- tableMusic(data)
}
## if try-error or musicTable==NULL, try _(band)
if(class(data)=="try-error" | is.null(data) | length(data)<1) {
url <- paste("https://en.wikipedia.org/wiki/", artist, "_(band)", sep="")
data <- try(url %>% html %>% html_nodes(xpath='//*[#id="mw-content-text"]/table[1]') %>% html_table(fill=T), silent=T)
if(class(data)=="try-error"){
data <- NULL
} else {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
data <- tableMusic(data)
}
} else {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
}
## if try-error or musicTable==NULL, try _(musician)
if(class(data)=="try-error" | is.null(data) | length(data)<1) {
url <- paste("https://en.wikipedia.org/wiki/", artist, "_(musician)", sep="")
data <- try(url %>% html %>% html_nodes(xpath='//*[#id="mw-content-text"]/table[1]') %>% html_table(fill=T), silent=T)
if(class(data)=="try-error"){
data <- NULL
} else {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
data <- tableMusic(data)
}
} else {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
}
data
}
## first try finding data
data <- artistData(artist)
## try finding with and/&
if(is.null(data)){data <- artistData(unlist(strsplit(artist, " and| &"))[1])}
## if no matches return ""
if(class(data)=="try-error" | is.null(data)) {
data <- ""
return()
} else {
if(class(data)!="data.frame") {data <- data.frame(data, stringsAsFactors=F)}
}
## if we have a matching page, pull the relevant data
origin <- data[data[,1]=="Origin",2]
if(length(origin)>0) {
home <- origin
} else {
born <- data[data[,1]=="Born",2]
if (length(born)>0) {
home <- unlist(strsplit(born, "age.[0-9]+)"))[2]
} else {
home <- ""
}
}
home
}
findHome("randy newman")

Answer 1

I figured it out. I had to add a url parameter into the tableMusic() function. As it was, it recycling the url from past searches. Thanks for the suggestion.