I use getSymbols to get the return data of sp500 in R,but i can't use the xts data in Ad() function,here is my code:
identifiers <- read.csv("sp500.csv",header = FALSE)
getSymbols("GOOG", src = "yahoo", from = "2013-01-02") #Stock: Google
LogRetGOOG = drop(coredata(na.omit(diff(log(lag(Ad(GOOG)))))))
stocks = matrix(nrow = length(LogRetGOOG),ncol = nrow(identifiers))
for (sedol in identifiers$V1) {
getSymbols(sedol, src = "yahoo", from = "2018-01-02") #Stock: Google
# dealead_data <- drop(coredata(na.omit(diff(log(lag(Ad(sedol)))))))
# stocks[,i] = dealead_data
# i <- i + 1
}
i <- 1
for (sedol in objs) {
x <- get(sedol)
# getSymbols(sedol, src = "yahoo", from = "2018-01-02") #Stock:
Google
dealead_data <-
drop(coredata(na.omit(diff(log(lag(as.name(sedol)))))))
stocks[,i] = dealead_data
i <- i + 1
}
the return result of as.name() is charactor,but I need the xts objcet,I don't know how to get it,thanks.
getSymbols(identifier, src = "yahoo", from = "2007-01-01",to = "2016-12-31",auto.assign = FALSE)
add auto.assign = FALSE,the issue was sovled
Related
I use the UN Comtrade data API with R.
library(rjson)
get.Comtrade <- function(url="http://comtrade.un.org/api/get?"
,maxrec=50000
,type="C"
,freq="A"
,px="HS"
,ps="now"
,r
,p
,rg="all"
,cc="TOTAL"
,fmt="json"
)
{
string<- paste(url
,"max=",maxrec,"&" #maximum no. of records returned
,"type=",type,"&" #type of trade (c=commodities)
,"freq=",freq,"&" #frequency
,"px=",px,"&" #classification
,"ps=",ps,"&" #time period
,"r=",r,"&" #reporting area
,"p=",p,"&" #partner country
,"rg=",rg,"&" #trade flow
,"cc=",cc,"&" #classification code
,"fmt=",fmt #Format
,sep = ""
)
if(fmt == "csv") {
raw.data<- read.csv(string,header=TRUE)
return(list(validation=NULL, data=raw.data))
} else {
if(fmt == "json" ) {
raw.data<- fromJSON(file=string)
data<- raw.data$dataset
validation<- unlist(raw.data$validation, recursive=TRUE)
ndata<- NULL
if(length(data)> 0) {
var.names<- names(data[[1]])
data<- as.data.frame(t( sapply(data,rbind)))
ndata<- NULL
for(i in 1:ncol(data)){
data[sapply(data[,i],is.null),i]<- NA
ndata<- cbind(ndata, unlist(data[,i]))
}
ndata<- as.data.frame(ndata)
colnames(ndata)<- var.names
}
return(list(validation=validation,data =ndata))
}
}
}
However, sometimes it fails to connect server and I need to run the code several times to start working. Solution given here, to use Retry() function, which retries a request until it succeeds, seems attractive.
However, I have some difficulties implementing this function in the code given above. has anybody used it before and knows how to recode it?
An API call using httr::RETRY could look like the following:
library(httr)
library(jsonlite)
res <- RETRY(
verb = "GET",
url = "http://comtrade.un.org/",
path = "api/get",
encode = "json",
times = 3,
query = list(
max = 50000,
type = "C",
freq = "A",
px = "HS",
ps = "now",
r = 842,
p = "124,484",
rg = "all",
cc = "TOTAL",
fmt = "json"
)
)
# alternativ: returns dataset as a `list`:
# parsed_content <- content(res, as = "parsed")
# returns dataset as a `data.frame`:
json_content <- content(res, as = "text")
parsed_content <- parse_json(json_content, simplifyVector = TRUE)
parsed_content$validation
parsed_content$dataset
I'd suggest rewriting the get.Comtrade function using httr:
get.Comtrade <- function(verb = "GET",
url = "http://comtrade.un.org/",
path = "api/get",
encode = "json",
times = 3,
max = 50000,
type = "C",
freq = "A",
px = "HS",
ps = "now",
r,
p,
rg = "all",
cc = "TOTAL",
fmt = "json") {
res <- httr::RETRY(
verb = verb,
url = url,
path = path,
encode = encode,
times = times,
query = list(
max = max,
type = type,
freq = freq,
px = px,
ps = ps,
r = r,
p = p,
rg = rg,
cc = cc,
fmt = fmt
)
)
jsonlite::parse_json(content(res, as = "text"), simplifyVector = TRUE)
}
s1 <- get.Comtrade(r = "842", p = "124,484", times = 5)
print(s1)
Please see this and this for more information on library(httr).
I have a nested list structure with some elements (not all) having attributes that I want to keep (I've converted some xml output to a list). I'm trying to flatten it into a data.frame. The structure is something like this:
myList <- structure(list(address = structure(list(Address = list(Line = list("xxxxxxx"),
Line = list("xxxxxxx"), Line = list("xxxxxxx"), PostCode = list(
"XXX XXX"))), type = "Residential", verified = "Unverified"),
amount = structure(list(paymentAmount = list(maxAmount = list(
amountPart = structure(list(Amount = list("0.00")), component = "Standard"),
amountPart = structure(list(Amount = list("0.00")), component = "Thing1"),
amountPart = structure(list(Amount = list("0.00")), component = "Thing2"),
amountPart = structure(list(Amount = list("0.00")), component = "Thing3"),
amountPart = structure(list(Amount = list("0.00")), component = "Thing4"),
amountPart = structure(list(Amount = list("100.00")), component = "Thing5"),
amountPart = structure(list(Amount = list("0.00")), component = "Thing6")),
otherAmount = list(Amount = list("0.00")),
discount = list("0.00"),
transition = list(
"0.00"), discounts = list(), regularPayment = list(
"200.00")),
paymentInfo = list(income = structure(list(
net = list("0")), refNumber = "xxxxxxx"))),
paymentDate = "2021-03-22", startDate = "2021-02-16", endDate = "2021-03-15")),
type = "Normal")
I've tried rapply(myList, attributes) but that just seems to return NULL.
I've also tried using a loop in a recursive function:
get_attributes <- function(myList, attribute_list = NULL) {
if (is.null(attribute_list)) attribute_list <- list()
for (i in seq_along(myList)) {
if (is.list(myList[[i]])) {
attribute_list <- c(attribute_list, sapply(myList[[i]], attributes))
attribute_list <- get_attributes(myList[[i]], attribute_list)
} else {
attribute_list <- c(attribute_list, attributes(myList[[i]]))
}
}
attribute_list
}
Once I've got the list of attributes, I then want to put them in a one row data.frame - something like data.frame(address.type = "Residential", address.verified = "Unverified", component.1 = "Standard", component.2 = "Thing1"
The function with a loop is a bit messy and not very 'R', and it also seems to spit out lots of repeated elements that I don't want. Does anyone have any idea how to implement this more elegantly?
UPDATE
I've refined the loop implementation to this, which seems to work, but I just couldn't figure out how to use either purrr or one of the *apply functions in place of the loop:
get_attributes <- function(myList, attribute_list = NULL, prefix = NULL) {
if (is.null(attribute_list)) {
attribute_list <- list()
}
if (is.null(prefix)) {
prefix <- ""
}
for (i in seq_along(myList)) {
name <- names(myList)[i]
attrs <- attributes(myList[[i]])
if (!is.null(attrs)) {
names(attrs) <- paste0(prefix, name, ".", names(attrs))
attrs <- attrs[!grepl("\\.names$", names(attrs))]
attribute_list <- c(attribute_list, attrs)
}
if (is.list(myList[[i]])) {
attribute_list <- get_attributes(myList[[i]],
attribute_list,
paste0(prefix, name, "."))
}
}
attribute_list
}
do.call(data.frame, get_attributes(myList))
You can gather all the attributes available and just keep the ones you are interested from it.
library(purrr)
map_df(myList, ~map_chr(attributes(.x), toString))
# names type verified paymentDate startDate endDate
# <chr> <chr> <chr> <chr> <chr> <chr>
#1 Address Residential Unverified NA NA NA
#2 paymentAmount, paymentInfo NA NA 2021-03-22 2021-02-16 2021-03-15
I am trying to load a few data with a nested-loop using pageviews. You already helped me get this result:
library("pageviews")
lang = c("it.wikipedia")
bm = c("ECB","Christine Lagarde")
x <- list(
list(),
list(),
list(),
list(),
list()
) # store results
for (i in seq_along(lang)) {
for (j in seq_along(bm)) {
x[[i]][[j]] = article_pageviews(project = lang[i], article = bm[j], platform = "all", user_type = "user", start = "2015100100", end = today(), reformat = TRUE, granularity = "daily")
}
}
The last step I need to do, however, involves reading some article for which project doesn't exist. Find an example below:
lang = c("it.wikipedia")
bm = c("Philip Lane")
x = article_pageviews(project = lang, article = bm, platform = "all", user_type = "user", start = "2015100100", end = today(), reformat = TRUE, granularity = "daily")
# Error in FUN(X[[i]], ...) :
# The date(s) you used are valid, but we either do not have data for those date(s), or the project you asked for is not loaded yet. Please check https://wikimedia.org/api/rest_v1/?doc for more information.
I would like to add this to the loop. I tried a few solutions but I don't manage to make the loop skip over if there is an error. I post below one mistaken attempt:
lang = c("it.wikipedia")
bm = c("ECB", "Christine Lagarde", "Philip Lane")
for (i in seq_along(lang)) {
for (j in seq_along(bm)) {
skip_to_next <- FALSE
tryCatch(x[[i]][[j]] = article_pageviews(project = lang[i], article = bm[j], platform = "all", user_type = "user", start = "2015100100", end = today(), reformat = TRUE, granularity = "daily"), error = function(e) {skip_to_next <<- TRUE})
if(skip_to_next) { next }
}
}
Can anyone help me run the loop and skip whenever it meets an error?
Thanks a lot!
You can use tryCatch as :
library(pageviews)
library(purrr)
lang = c("it.wikipedia")
bm = c("ECB", "Christine Lagarde", "Philip Lane")
map_df(lang, function(x) map_df(bm, function(y)
tryCatch(article_pageviews(project = x, article = y, platform = "all", user_type = "user", start = "2015100100", end = today(), reformat = TRUE, granularity = "daily"),
error = function(e) {}))) -> result
I am extracting twitter data using the twitteR package and storing them in a dataframe x.
I first created the dataframe.
x <- data.frame(
name = character(),
screen_name = character(),
id = integer(),
description = character(),
statuses_count = integer(),
followersCount = integer(),
favoritesCount = integer(),
friendsCount = integer(),
url = character(),
created = integer(),
verified = integer(),
profile_image_url = character(),
stringsAsFactors=FALSE
)
Then created a function to return the data of a specific user
adduserdata <- function(username = ""){
user <- getUser(username)
userdata = c(name = user$name,
screen_name = user$screenName,
id = user$id,
description = user$description,
statuses_count = user$statusesCount,
followersCount = user$followersCount,
favoritesCount = user$favoritesCount,
friendsCount = user$friendsCount,
url = user$url,
created = user$created,
verified = user$verified,
profile_image_url = user$profileImageUrl)
return(userdata)
}
I now want to get the data of each user in the list ns and append them to the dataframe x
ns <- c("realDonaldTrump","BarackObama")
for (n in ns) {
user <- adduserdata(n)
x <- bind(x, user)
}
But I get an error stating 'invalid factor level'. I'm not sure why.
Return a dataframe from addUser function.
adduserdata <- function(username = ""){
user <- getUser(username)
userdata = data.frame(name = user$name,
screen_name = user$screenName,
id = user$id,
description = user$description,
statuses_count = user$statusesCount,
followersCount = user$followersCount,
favoritesCount = user$favoritesCount,
friendsCount = user$friendsCount,
url = user$url,
created = user$created,
verified = user$verified,
profile_image_url = user$profileImageUrl)
return(userdata)
}
and try :
result <- do.call(rbind, lapply(ns, adduserdata))
Or
result <- purrr::map_df(ns, adduserdata)
I don't know why but I am facing errors when trying to download data from "Oanda" using getSymbols function:
library(quantmod)
require(quantmod)
tickers <- c("USD/EUR","EUR/BRL","USD/ARS","USD/BRL","USD/CLP","USD/COP")
tickers2 <- c("USDEUR","EURBRL","USDARS","USDBRL","USDBRL","USDCLP","USDCOP")
getSymbols(tickers, src = "oanda", env = parent.frame(n=1),
from = "2014-01-01", to = Sys.Date(), auto.assign = TRUE,
warning = FALSE)
Closeprices <- do.call(cbind, lapply(tickers2, function(x) get(x)))
head(Closeprices)
dat1 <- as.data.frame(Closeprices)
dat1$date <- time(Closeprices)
head(dat1)
and it returns the following error:
> getSymbols("USD/EUR", src = "oanda", env = parent.frame(n=1),
+ to = Sys.Date(), auto.assign = TRUE,
+ warning = FALSE)
Error in charToDate(x) :
character string is not in a standard unambiguous format
It used to work well, but now it is crashing.
Can you help me out?