Getting an error message when using rvest for webscraping? - r

Question

I want to scrape all the shapefiles from the following website: https://www.sciencebase.gov/catalog/items?q=&filter0=browseCategory%3DData&community=California+Condor&filter1=browseType%3DMap+Service&filter2=browseType%3DOGC+WMS+Layer&filter3=browseType%3DDownloadable&filter4=facets.facetName%3DShapefile&&filter5=browseType%3DShapefile
I used the following script:
installed.packages('rvest')
library(rvest)
library(tidyverse)
## CONSTANT ----
URL <- "https://www.sciencebase.gov/catalog/item/54471eb5e4b0f888a81b82ca"
dir_out <- "~/condor"
## MAIN ----
# Get the webpage content
webpage <- read_html(URL)
# Extract the information of interest from the website
data <- html_nodes(webpage, ".sb-file-get sb-download-link")
# Grab the base URLs to download all the referenced data
url_base <- html_attr(data,"href")
# Filter the zip files
shapefile_base <- grep("*.zip",url_base, value=TRUE)
# Fix the double `//`
shapefile_fixed <- gsub("//", "/", shapefile_base)
# Add the URL prefix
shapefile_full <- paste0("https://www.sciencebase.gov/",shapefile_fixed)
# Create the output directory
dir.create(dir_out, showWarnings = FALSE)
# Create a list of filenames
filenames_full <- file.path(dir_out,basename(shapefile_full))
# Download the files
lapply(shapefile_full, FUN=function(x) download.file(x, file.path(dir_out,basename(x))))
# Unzip the files
unzip(filenames_full, overwrite = TRUE)
However for the download file section, I get the following error
> lapply(shapefile_full, FUN=function(x) download.file(x, file.path(dir_out,basename(x))))
trying URL 'https://www.sciencebase.gov/'
Content type 'text/html' length unknown
downloaded 111 bytes
[[1]]
[1] 0
> # Unzip the files
> unzip(filenames_full, overwrite = TRUE)
Warning message:
In unzip(filenames_full, overwrite = TRUE) :
error 1 in extracting from zip file

Answer 1

This works for me and produces 6 zip files in my working directory.
library(rvest)
URL <- "https://www.sciencebase.gov/catalog/item/54471eb5e4b0f888a81b82ca"
webpage <- read_html(URL)
# Extract the information of interest from the website
data <- html_nodes(webpage, "span.sb-file-get")
# Grab the base URLs to download all the referenced data
url_base <- html_attr(data,"data-url")
# Filter the zip files
shapefile_full <- paste0("https://www.sciencebase.gov",url_base)
Map(download.file, shapefile_full, sprintf('file%d.zip', seq_along(shapefile_full)))