Im some ocassion a Stack user help me for make this script. Im edit it for add more attributes but I have problems when try to add Authors
The Author label is next to target and href. I have problem in this part.
library(tidyverse)
library(rvest)
startTime <- Sys.time()
get_cg <- function(pages) {
cat("Scraping page", pages, "\n")
page <-
str_c("https://cgspace.cgiar.org/discover?
scope=10568%2F106146&query=cassava&submit=&rpp=10&page=", pages) %>%
read_html()
tibble(
title = page %>%
html_elements(".ds-artifact-item") %>%
html_element(".description-info") %>%
html_text2(), # run well
fecha = page %>%
html_elements(".ds-artifact-item") %>%
html_element(".date") %>%
html_text2(), # run well
Type = page %>%
html_elements(".ds-artifact-item") %>%
html_element(".artifact-type") %>%
html_text2(), # run well
Autor= page %>%
html_elements(".ds-artifact-item") %>%
html_element(".description-info") %>%
html_attr("href"), # not download the Authors
link = page %>%
html_elements(".ds-artifact-item") %>%
html_element(".description-info") %>%
html_attr("href") %>% # run well
str_c("https://cgspace.cgiar.org", .)
)
}
df <- map_dfr(1, get_cg)
endTime <- Sys.time()
print(endTime - startTim)
Im try with other selector but get NA
This should get you a collapsed list of authors for each book, separated by ; , basically the same as presented on the page:
library(tidyverse, warn.conflicts = F)
library(rvest, warn.conflicts = F)
startTime <- Sys.time()
get_cg <- function(pages) {
cat("Scraping page", pages, "\n")
page <-
str_c("https://cgspace.cgiar.org/discover?scope=10568%2F106146&query=cassava&submit=&rpp=10&page=", pages) %>%
read_html()
html_elements(page, "div.artifact-description > div.artifact-description") %>%
map_df(~ list(
title = html_element(.x, ".description-info") %>% html_text2(),
fecha = html_element(.x, ".date") %>% html_text2(),
Type = html_element(.x, ".artifact-type") %>% html_text2(),
# Autor_links = html_elements(.x,".description-info > span > a") %>% html_attr("href") %>% paste(collapse = ";"),
Autor = html_element(.x, "span.description-info") %>% html_text2(),
link = html_element(.x, "a.description-info") %>% html_attr("href") %>% str_c("https://cgspace.cgiar.org", .)
))
}
df <- map_dfr(1, get_cg)
#> Scraping page 1
endTime <- Sys.time()
print(endTime - startTime)
#> Time difference of 0.989037 secs
Result:
df
#> # A tibble: 10 × 5
#> title fecha Type Autor link
#> <chr> <chr> <chr> <chr> <chr>
#> 1 Global Climate Regions for Cassava 2020… Type… Hyma… http…
#> 2 Performance of the CSM–MANIHOT–Cassava model for sim… 2021… Type… Phon… http…
#> 3 Adoption of cassava improved modern varieties in the… 2020 Type… Laba… http…
#> 4 First report of Sri Lankan cassava mosaic virus and … 2021… Type… Chit… http…
#> 5 Surveillance and diagnostics dataset on Sri Lankan c… 2020 Type… Siri… http…
#> 6 Socieconomic and soil conservation practices for cas… 2022… Type… Ibar… http…
#> 7 The transformation and outcome of traditional cassav… 2020 Type… Dou,… http…
#> 8 Cassava Annual Report 2019 2020 Type… Inte… http…
#> 9 Cassava Annual Report 2020 2021… Type… Bece… http…
#> 10 Adoption of cassava improved modern varieties in the… 2020 Type… Flor… http…
glimpse(df)
#> Rows: 10
#> Columns: 5
#> $ title <chr> "Global Climate Regions for Cassava", "Performance of the CSM–MA…
#> $ fecha <chr> "2020-08-03", "2021-05-01", "2020", "2021-04-23", "2020", "2022-…
#> $ Type <chr> "Type:Dataset", "Type:Journal Article", "Type:Dataset", "Type:Jo…
#> $ Autor <chr> "Hyman, Glenn G.", "Phoncharoen, Phanupong; Banterng, Poramate; …
#> $ link <chr> "https://cgspace.cgiar.org/handle/10568/109500", "https://cgspac…
Created on 2022-12-03 with reprex v2.0.2
Related
I am trying to download several tables from this website using rvest and SelectorGadget.
The css selector is "#main li" as can be seen from the screenshot below.
When I run the following code, unfortunately an empty table results.
library(rvest)
psh <- read_html("https://pershingsquareholdings.com/performance/net-asset-value-and-returns/")
psh.node <- html_node(psh, "#main li")
psh.table = html_table(psh, fill = TRUE)
I guess the site prevents scraping, but it would otherwise be great if an alternative way could be recommended to get the data.
Thanks in advance!
Problem is that it is not an html <table> but a list:
library(rvest)
library(purrr)
psh <- read_html("https://pershingsquareholdings.com/performance/net-asset-value-and-returns/")
psh.node <- html_node(psh, "#main .psh_table")
headers <- psh.node %>% html_element(".psh_table_row.headings") %>%
html_elements("li") %>% html_text()
table <- psh.node %>% html_elements("ul.psh_table_row") %>%
map_dfr(~ html_elements(., "li") %>% html_text() %>% set_names(headers))
table
#> # A tibble: 44 × 10
#> `As of Date` Period USDNAV…¹ Euron…² GBPNA…³ LSE G…⁴ LSE U…⁵ MTDRe…⁶ QTDRe…⁷
#> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr>
#> 1 As of Date Period USDNAV/… Eurone… GBPNAV… LSE GB… LSE US… MTDRet… QTDRet…
#> 2 25 October Weekly $51.01 $32.60 £44.47 £28.45 $32.63 12.0% 12.0%
#> 3 18 October Weekly $47.81 $30.25 £42.22 £26.70 $30.23 5.0% 5.0%
#> 4 11 October Weekly $44.96 $29.35 £40.93 £26.35 $29.55 -1.3% -1.3%
#> 5 30 September Monthly $45.55 $30.00 £40.79 £27.00 $30.25 -4.5% 8.2%
#> 6 27 September Weekly $45.49 $30.25 £42.44 £28.00 $30.23 -4.6% 8.1%
#> 7 20 September Weekly $47.88 $32.00 £42.06 £27.80 $31.75 0.4% 13.8%
#> 8 13 September Weekly $49.11 $32.00 £42.71 £27.70 $32.13 2.7% 16.4%
#> 9 6 September Weekly $47.80 $32.25 £41.51 £28.10 $32.48 -0.1% 13.3%
#> 10 31 August Monthly $47.83 $32.70 £41.17 £27.90 $32.83 2.9% 13.4%
#> # … with 34 more rows, 1 more variable: YTDReturn <chr>, and abbreviated
#> # variable names ¹`USDNAV/Share`, ²`EuronextPrice/Share`, ³`GBPNAV/Share`,
#> # ⁴`LSE GBPPrice/Share`, ⁵`LSE USDPrice/Share`, ⁶MTDReturn, ⁷QTDReturn
Edit
To find all similar tables:
psh.nodes <- html_elements(psh, "#main .psh_table")
tables <- map(psh.nodes, function(psh.node){
headers <- psh.node %>% html_element(".psh_table_row.headings") %>%
html_elements("li") %>% html_text()
table <- psh.node %>% html_elements("ul.psh_table_row") %>%
map_dfr(~ html_elements(., "li") %>% html_text() %>% set_names(headers))
table
})
I am an infectious diseases physician and have set myself the challenge of creating a dataframe with the UK cumulative published cases of monkeypox, so I can graph it as a runing tally or a chloropleth map as there is no nice dashboard at present for this.
All the data is published as html webpages rather than as a nice csv so I am trying to scrape it all off the internet using the rvest package.
Data is only published intermittently (about twice per week) with the cumulative totals for each of the 4 home nations in UK.
I have managed to get working code to pull data from each of the separate webpages and testing it on the first 2 pages in my mpx_gov_uk_pages list works well giving a small example tibble:
library(tidyverse)
library(lubridate)
library(rvest)
library(janitor)
# load in overview page url which has links to each date of published cases
mpx_gov_uk_overview_page <- c("https://www.gov.uk/government/publications/monkeypox-outbreak-epidemiological-overview")
# extract urls for each date page
mpx_gov_uk_pages <- mpx_gov_uk_overview_page %>%
read_html %>%
html_nodes(".govuk-link") %>%
html_attr('href') %>%
str_subset("\\d{1,2}-[a-z]+-\\d{4}") %>%
paste0("https://www.gov.uk", .) %>%
as.character()
# make table for home nations for each date
table1 <- mpx_gov_uk_pages[1] %>%
read_html() %>%
html_table() %>%
.[[1]] %>%
janitor::clean_names() %>%
rename(area = starts_with(c("uk", "devolved")),
cases = matches(c("total", "confirmed_cases"))) %>%
separate(cases, c("cases", NA), sep = "\\s\\(") %>%
mutate(date = dmy(str_extract(mpx_gov_uk_pages[1], "\\d{1,2}-[a-z]+-\\d{4}")),
cases = as.numeric(gsub(",", "", cases))) %>%
select(date, area, cases) %>%
filter(!area %in% c("Total"))
table2 <- mpx_gov_uk_pages[2] %>%
read_html() %>%
html_table() %>%
.[[1]] %>%
janitor::clean_names() %>%
rename(area = starts_with(c("uk", "devolved")),
cases = matches(c("total", "confirmed_cases"))) %>%
separate(cases, c("cases", NA), sep = "\\s\\(") %>%
mutate(date = dmy(str_extract(mpx_gov_uk_pages[2], "\\d{1,2}-[a-z]+-\\d{4}")),
cases = as.numeric(gsub(",", "", cases))) %>%
select(date, area, cases) %>%
filter(!area %in% c("Total"))
#> Warning: Expected 2 pieces. Missing pieces filled with `NA` in 1 rows [4].
# Combine tables
bind_rows(table1, table2)
#> # A tibble: 8 × 3
#> date area cases
#> <date> <chr> <dbl>
#> 1 2022-08-02 England 2638
#> 2 2022-08-02 Northern Ireland 24
#> 3 2022-08-02 Scotland 65
#> 4 2022-08-02 Wales 32
#> 5 2022-07-29 England 2436
#> 6 2022-07-29 Northern Ireland 19
#> 7 2022-07-29 Scotland 61
#> 8 2022-07-29 Wales 30
I want to automate this by creating a generic function and passing the list of urls to purrr::map_df as there will be an ever growing number of pages (there's already 13):
pull_first_table <- function(x){
x %>%
read_html() %>%
html_table() %>%
.[[1]] %>%
janitor::clean_names() %>%
rename(area = starts_with(c("uk", "devolved")),
cases = matches(c("total", "confirmed_cases"))) %>%
separate(cases, c("cases", NA), sep = "\\s\\(") %>%
mutate(date = dmy(str_extract({{x}}, "\\d{1,2}-[a-z]+-\\d{4}")),
cases = as.numeric(gsub(",", "", cases))) %>%
select(date, area, cases) %>%
filter(!area %in% c("Total"))
}
summary_table <- map_df(mpx_gov_uk_pages, ~ pull_first_table)
Error in `dplyr::bind_rows()`:
! Argument 1 must be a data frame or a named atomic vector.
Run `rlang::last_error()` to see where the error occurred.
The generic function seems to work ok when I supply it with a single element e.g. mpx_gov_uk_cases[2] but I cannot seem to get map_df to work properly even though the webscraping is producing tibbles.
All help and pointers greatly welcomed.
We just need the function and not a lambda expression.
map_dfr(mpx_gov_uk_pages, pull_first_table)
-output
# A tibble: 52 × 3
date area cases
<date> <chr> <dbl>
1 2022-08-02 England 2638
2 2022-08-02 Northern Ireland 24
3 2022-08-02 Scotland 65
4 2022-08-02 Wales 32
5 2022-07-29 England 2436
6 2022-07-29 Northern Ireland 19
7 2022-07-29 Scotland 61
8 2022-07-29 Wales 30
9 2022-07-26 England 2325
10 2022-07-26 Northern Ireland 18
# … with 42 more rows
If we use the lambda expression,
map_dfr(mpx_gov_uk_pages, ~ pull_first_table(.x))
I would like to scrape a website storing German cycling results, but I'm struggling to get the urls pointing to the race result.
Website with result table
This is what I got so far, to me the html table seems also quite oddly formatted, but that could also be due to my lack of html knowledge:
library(tidyverse)
library(magrittr)
library(rvest)
#read html
result_url <- "https://www.rad-net.de/rad-net-ergebnisse.htm?name=Ausschreibung&view=ascr_erg&rnswp_disziplin=1"
results <- read_html(result_url)
#extract date, race name
results %>%
html_table(header = T, fill = T) %>%
extract2(8) %>%
tibble()
#> # A tibble: 40 x 2
#> Datum Veranstaltungstitel
#> <chr> <chr>
#> 1 So, 19.07.20… "5. Rosenheimer Jugend - Kriterium"
#> 2 So, 12.07.20… "Swiss O Par Preis"
#> 3 So, 12.07.20… "Deutsche Meisterschaft Einzelzeitfahren U19m/w"
#> 4 So, 12.07.20… "Jugendrenntag der RV Offenbach"
#> 5 Sa, 04.07.20… "CoronaChronoNRW"
#> 6 Sa, 20.06.20… "GCA Klassiker PROS / KIDS / ALL powered by «Müller \u0096 Die…
#> 7 Sa, 13.06.20… "GCA Klassiker PROS / KIDS / ALL powered by «Müller \u0096 Die…
#> 8 Sa, 06.06.20… "GCA Klassiker PROS / KIDS / ALL powered by «Müller \u0096 Die…
#> 9 So, 31.05.20… "Westsachsenklassiker - 72. Sachsenringradrennen"
#> 10 So, 08.03.20… "8. Herforder Frühjahrspreis"
#> # … with 30 more rows
Created on 2020-07-25 by the reprex package (v0.3.0)
I think you're looking for a bit more information than is normally provided by the html_table function (there are actually several nested html tables on the page anyway). I think this is what you are looking for:
library(tidyverse)
library(magrittr)
library(rvest)
results <- paste0("https://www.rad-net.de/rad-net-ergebnisse.htm",
"?name=Ausschreibung&view=ascr_erg&rnswp_disziplin=1") %>%
read_html()
link_nodes <- results %>% html_nodes(xpath = "//table//a")
link_text <- link_nodes %>% html_text()
index <- (which(link_text == "hier") + 1):(which(link_text == "N\u00e4chste") - 1)
link_nodes <- link_nodes[index]
dates <- link_nodes %>%
html_nodes(xpath = "//table//a/parent::td/preceding-sibling::td/font") %>%
html_text()
df <- tibble(Datum = dates[-1],
Veranstaltungstitel = link_nodes %>% html_text(),
link = link_nodes %>% html_attr("href"))
df
#> # A tibble: 40 x 3
#> Datum Veranstaltungstitel link
#> <chr> <chr> <chr>
#> 1 So, 19.0~ "5. Rosenheimer Jugend - Kriterium" /rad-net-portal/rad-net-erge~
#> 2 So, 12.0~ "Swiss O Par Preis" /rad-net-portal/rad-net-erge~
#> 3 So, 12.0~ "Deutsche Meisterschaft Einzelzeitfa~ /rad-net-portal/rad-net-erge~
#> 4 So, 12.0~ "Jugendrenntag der RV Offenbach" /rad-net-portal/rad-net-erge~
#> 5 Sa, 04.0~ "CoronaChronoNRW" /rad-net-portal/rad-net-erge~
#> 6 Sa, 20.0~ "GCA Klassiker PROS / KIDS / ALL pow~ /rad-net-portal/rad-net-erge~
#> 7 Sa, 13.0~ "GCA Klassiker PROS / KIDS / ALL pow~ /rad-net-portal/rad-net-erge~
#> 8 Sa, 06.0~ "GCA Klassiker PROS / KIDS / ALL pow~ /rad-net-portal/rad-net-erge~
#> 9 So, 31.0~ "Westsachsenklassiker - 72. Sachsenr~ /rad-net-portal/rad-net-erge~
#> 10 So, 08.0~ "8. Herforder Frühjahrspreis" /rad-net-portal/rad-net-erge~
#> # ... with 30 more rows
Created on 2020-07-25 by the reprex package (v0.3.0)
I've never scraped. Would it be straightforward to scrape the text in the main, big gray box only from the link below (starting with header SRUS43 KMSR 271039, ending with .END)? My end goal is to basically have three tidy columns of data from all that text: the five digit codes, the values in inches, and the basin elevation descriptions, so any pointers with processing the text format are welcome, too.
https://www.nohrsc.noaa.gov/shef_archive/index.html?rfc=cnrfc&product=swe&year=2019&month=3&day=27&hour=6
thank you for any help.
Reading in the text is fairly easy (see #DiceBoyT answer). Cleaning up the format for three columns is a bit more involved. Below could use some clean-up (especially with the regex), but it gets the job done:
library(tidyverse)
library(rvest)
text <- read_html("https://www.nohrsc.noaa.gov/shef_archive/index.html?rfc=cnrfc&product=swe&year=2019&month=3&day=27&hour=6") %>%
html_node(".notes") %>%
html_text()
df <- tibble(txt = read_lines(text))
df %>%
mutate(
row = row_number(),
with_code = str_extract(txt, "^[A-z0-9]{5}\\s+\\d+(\\.)?\\d"),
wo_code = str_extract(txt, "^:?\\s+\\d+(\\.)?\\d") %>% str_extract("[:digit:]+\\.?[:digit:]"),
basin_desc = if_else(!is.na(with_code), lag(txt, 1), NA_character_) %>% str_sub(start = 2)
) %>%
separate(with_code, c("code", "val"), sep = "\\s+") %>%
mutate(
combined_val = case_when(
!is.na(val) ~ val,
!is.na(wo_code) ~ wo_code,
TRUE ~ NA_character_
) %>% as.numeric
) %>%
filter(!is.na(combined_val)) %>%
mutate(
code = zoo::na.locf(code),
basin_desc = zoo::na.locf(basin_desc)
) %>%
select(
code, combined_val, basin_desc
)
#> # A tibble: 643 x 3
#> code combined_val basin_desc
#> <chr> <dbl> <chr>
#> 1 ACSC1 0 San Antonio Ck - Sunol
#> 2 ADLC1 0 Arroyo De La Laguna
#> 3 ADOC1 0 Santa Ana R - Prado Dam
#> 4 AHOC1 0 Arroyo Honda nr San Jose
#> 5 AKYC1 41 SF American nr Kyburz
#> 6 AKYC1 3.2 SF American nr Kyburz
#> 7 AKYC1 42.2 SF American nr Kyburz
#> 8 ALQC1 0 Alamo Canal nr Pleasanton
#> 9 ALRC1 0 Alamitos Ck - Almaden Res
#> 10 ANDC1 0 Coyote Ck - Anderson Res
#> # ... with 633 more rows
Created on 2019-03-27 by the reprex package (v0.2.1)
This is pretty straightforward to scrape with rvest:
library(rvest)
text <- read_html("https://www.nohrsc.noaa.gov/shef_archive/index.html?rfc=cnrfc&product=swe&year=2019&month=3&day=27&hour=6") %>%
html_node(".notes") %>%
html_text()
I'm trying to write a script that will go through a list of players provided by the website Transfermarkt and gathering some information about them. For that, I've created the script below, but faced a problem with 1 of the 29 players in the list. Due to one page being arranged differently than the others, the code outputs a list of only 28 players since it can't find information on the aforementioned page.
I understand why the code I've written doesn't find any information on the given page and thus giving me a list of 28, but I don't know how to rewrite a code in order to achieve what I want:
for the script to simply replace the entry with a "-" if it does not find anything, in this case a nationality, for the node on a particular page and return a full list with 29 players with all the other info in it.
The player page in question is this and while the other pages has the node used in the code for nationality, here it's ".dataValue span".
I'm still quite new to R and it might be an easy fix, but atm I can't figure it out. Any help or advise is appreciated.
URL <- "http://www.transfermarkt.de/fc-bayern-munchen/leistungsdaten/verein/27/reldata/%262016/plus/1"
WS <- read_html(URL)
Team <- WS %>% html_nodes(".spielprofil_tooltip") %>% html_attr("href") %>% as.character()
Team <- paste0("http://www.transfermarkt.de",Team)
Catcher <- data.frame(Name=character(),Nat=character(),Vertrag=character())
for (i in Team) {
WS1 <- read_html(i)
Name <- WS1 %>% html_nodes("h1") %>% html_text() %>% as.character()
Nat <- WS1 %>% html_nodes(".hide-for-small+ p .dataValue span") %>% html_text() %>% as.character()
Vertrag <- WS1 %>% html_nodes(".dataValue:nth-child(9)") %>% html_text() %>% as.character()
if (length(Nat) > 0) {
temp <- data.frame(Name,Nat,Vertrag)
Catcher <- rbind(Catcher,temp)
}
else {}
cat("*")
}
num_Rows <- nrow(Catcher)
odd_indexes <- seq(1,num_Rows,2)
Catcher <- data.frame(Catcher[odd_indexes,])
It's honestly easier to scrape the whole table, just in case things move around. I find purrr is a helpful complement for rvest, allowing you to iterate over URLs and node lists and easily coerce results to data.frames:
library(rvest)
library(purrr)
# build dynamically if you like
urls <- c(boateng = 'http://www.transfermarkt.de/jerome-boateng/profil/spieler/26485',
friedl = 'http://www.transfermarkt.de/marco-friedl/profil/spieler/156990')
# scrape once, parse iteratively
html <- urls %>% map(read_html)
df <- html %>%
map(html_nodes, '.dataDaten p') %>%
map_df(map_df,
~list(
variable = .x %>% html_node('.dataItem') %>% html_text(trim = TRUE),
value = .x %>% html_node('.dataValue') %>% html_text(trim = TRUE) %>% gsub('\\s+', ' ', .)
),
.id = 'player')
df
#> # A tibble: 17 × 3
#> player variable value
#> <chr> <chr> <chr>
#> 1 boateng Geb./Alter: 03.09.1988 (28)
#> 2 boateng Geburtsort: Berlin
#> 3 boateng Nationalität: Deutschland
#> 4 boateng Größe: 1,92 m
#> 5 boateng Position: Innenverteidiger
#> 6 boateng Vertrag bis: 30.06.2021
#> 7 boateng Berater: SAM SPORTS
#> 8 boateng Nationalspieler: Deutschland
#> 9 boateng Länderspiele/Tore: 67/1
#> 10 friedl Geb./Alter: 16.03.1998 (19)
#> 11 friedl Nationalität: Österreich
#> 12 friedl Größe: 1,87 m
#> 13 friedl Position: Linker Verteidiger
#> 14 friedl Vertrag bis: 30.06.2021
#> 15 friedl Berater: acta7
#> 16 friedl Akt. Nationalspieler: Österreich U19
#> 17 friedl Länderspiele/Tore: 6/0
Alternately, that particular piece of data is in three places on those pages, so if one is inconsistent there's a chance the others are better. Or grab them from the table with the whole team—countries are not printed, but they're in the title attribute of the flag images, which can be grabbed with html_attr:
html <- read_html('http://www.transfermarkt.de/fc-bayern-munchen/leistungsdaten/verein/27/reldata/%262016/plus/1')
team <- html %>%
html_nodes('tr.odd, tr.even') %>%
map_df(~list(player = .x %>% html_node('a.spielprofil_tooltip') %>% html_text(),
nationality = .x %>% html_nodes('img.flaggenrahmen') %>% html_attr('title') %>% toString()))
team
#> # A tibble: 29 × 2
#> player nationality
#> <chr> <chr>
#> 1 Manuel Neuer Deutschland
#> 2 Sven Ulreich Deutschland
#> 3 Tom Starke Deutschland
#> 4 Jérôme Boateng Deutschland
#> 5 David Alaba Österreich
#> 6 Mats Hummels Deutschland
#> 7 Javi Martínez Spanien
#> 8 Juan Bernat Spanien
#> 9 Philipp Lahm Deutschland
#> 10 Rafinha Brasilien, Deutschland
#> # ... with 19 more rows