Related
I've started receiving 429 errors for the below script. The API I'm scraping requires a user-agent to be specified.
I'm at a loss for how do to specify a user agent header with the package I am using. The attempts I made using RCurl::getUrl produced errors as well.
Using options(HTTPUserAgent = "what google returns when I search my user agent") did not fix the 429 problem.
API documentation linked below.
https://docs.helium.com/api/blockchain/introduction/#specify-a-user-agent
library(jsonlite)
blocks_api <- 'https://api.helium.io/v1/blocks'
blocks <- fromJSON(blocks_api)
endTime <- Sys.Date()
blockMax_api <- paste0(blocks_api,"/height","/?max_time=",endTime)
blockMax_ep <- fromJSON(blockMax_api)
blockMax <- max(blockMax_ep$data$height)
startTime <- Sys.Date() - 1
blockMin_api <- paste0(blocks_api,"/height","/?max_time=",startTime)
blockMin_ep <- fromJSON(blockMin_api)
blockMin <- blockMin_ep$data$height
period_blocks <- blockMax - blockMin
blockTimes <- data.frame()
oraclePrice <- 'https://api.helium.io/v1/oracle/prices'
for(i in blockMin:blockMax){
block_n <- fromJSON(paste0(blocks_api,"/",i))
block_n <- as.data.frame(block_n)
block_n$data.time <- anytime(block_n$data.time)
block_n <- block_n[,c(2,5,6)]
oracleBlockPrice <- fromJSON(paste0(oraclePrice,"/",i))
block_n$HNTprice <- oracleBlockPrice$data$price / 100000000
blockTimes <- rbind(blockTimes,block_n)
Sys.sleep(1)
}
This is how the author of the jsonlite changes the user-agent in the fromJSON function. Change the useragent variable to the text that you want:
h <- curl::new_handle(useragent = paste("jsonlite /", R.version.string))
curl::handle_setheaders(h, Accept = "application/json, text/*, */*")
txt <- curl::curl(url, handle = h)
And then call fromJSON
fromJSON(txt)
We have to migrate to Oauth 2.0. However, I am facing the following issues. Status 400 I believe due to some parameters that may not be entered right. If anyone know what could be wrong or have any idea please let me know.
and this is my code:
pack <- c('curl','xml2','XML', 'plyr', 'dplyr','tidyr', 'httr', 'tools', 'lubridate',
'jsonlite', 'stringr', 'data.table', 'anytime')
sapply(pack, function(x){
if(!require(x,character.only = T, quietly = T)) {install.packages(x, quiet = T)}
require(x, quietly = T, character.only = T)
})
#New Xero & WFM Api OAuth 2.0 credentials
Client_ID <- 'YOUR_CLIENT_ID'
Client_secret<- 'YOUR_CLIENT_SECRET'
XTID <- 'YOUR_REFERAL_ID'#Referral_ID
Redirect_URI <- 'https://xero.com/' #OAuth 2.0 redirect URI
response <- GET(paste0('https://login.xero.com/identity/connect/authorize?response_type=code&client_id=',
Client_ID,'&redirect_uri=',Redirect_URI,'&scope=workflowmax%20offline_access'))
browseURL(response$url)
It works till here I am able to retrieve the first code, but I can't get the tokens
This is what Xero Doc said:
Xero Doc request:
POST https://identity.xero.com/connect/token
authorization: "Basic " + base64encode(client_id + ":" + client_secret)
Content-Type: application/x-www-form-urlencoded
grant_type=authorization_code
&code=xxxxxx
&redirect_uri=https://myapp.com/redirect
Now trying to get the token I tried several forms. however, they all look really similar.
code <- 'YOUR_CODE'
credentials = list();
credentials['grant_type'] <- "authorization_code"
credentials['code'] <- code
credentials['redirect_uri'] <- Redirect_URI
b64_id_secret <- base64_enc('YOUR_CLIENT_ID:YOUR_CLIENT_SECRET')
# user <- paste("Basic", user, sep = " ")
url <- 'https://identity.xero.com/connect/token'
POST(url, add_headers('Authorization'= paste0('Basic ', b64_id_secret)),
body = credentials,
verbose(), encode = 'form')
Response:
Response [https://identity.xero.com/connect/token?grant_type=authorization_code&code=YOUR_CODE&redirect_uri=https://waterlineprojects.com/]
Date: 2020-11-20 02:48
Status: 400
Content-Type: application/json; charset=UTF-8
Size: 27 B
XERO DOCUMENTATION: https://developer.xero.com/documentation/oauth2/overview
I did my workaround and got it running. This is the way to access the Oauth 2.0.
pack <- c('curl','xml2','XML', 'plyr', 'dplyr','tidyr', 'httr', 'tools', 'lubridate',
'jsonlite', 'stringr', 'data.table', 'anytime', 'RCurl', 'rvest', 'opnessl', 'jose')
sapply(pack, function(x){
if(!require(x,character.only = T, quietly = T)) {install.packages(x, quiet = T)}
require(x, quietly = T, character.only = T)
})
#New Xero & WFM Api OAuth 2.0 credentials
Client_ID <- 'YOUR_ID'
Client_secret<- 'YOUR_SECRET'
XTID_Xero <- 'YOUR_XTID'#Referral_ID
Redirect_URI <- 'YOUR_CALL_BACK_URL' #OAuth 2.0 redirect URI
# Create the app
app <- oauth_app("YOUR_API_NAME",
key = Client_ID,
secret = Client_secret,
redirect_uri = Redirect_URI
)
# Create the endpoint
create_endpoint <- function()
{
request <- "https://identity.xero.com/connect/token"
authorize <- "https://login.xero.com/identity/connect/authorize"
access <- "https://identity.xero.com/connect/token"
httr::oauth_endpoint(request, authorize, access)
}
api <- create_endpoint()
header <- httr::add_headers(Authorization=paste0("Basic ", RCurl::base64Encode(charToRaw(paste0(Client_ID, ":", Client_secret)))))
content_type <- httr::content_type("application/x-www-form-urlencoded")
# Define the scope
scope_WFM <- "openid profile offline_access payroll.employees.read payroll.payruns.read payroll.payslip.read payroll.timesheets.read accounting.transactions.read accounting.reports.read accounting.journals.read"
# Get the code
httr::BROWSE(oauth2.0_authorize_url(api, app, scope = scope_WFM))
#get the code from the URL displayed in your browser
code_xero <- 'YOR_CODE'
state_xero <- 'YOUR_STATE'
token <- httr::oauth2.0_token(
endpoint = api,
app = app,
scope = scope_WFM,
config_init = c(header, content_type),
use_basic_auth = TRUE,
query_authorize_extra = list(prompt = "login"),
type = "code",
credentials = oauth2.0_access_token(api, app, code_xero),
cache = FALSE
)
#get your xero-tenant-id
access <- GET("https://api.xero.com/connections", config = token)
connections <- content(access, 'text')
connections <- fromJSON(connections, flatten = T)
looks like you are close. Like MrFlick said without your client ID and starting a case with Xero API tech support (email api#xero.com with your client id and dat of log) its hard to know for sure.
One tip might be it, your redirect URI has to match exactly same value that is in your https://developer.xero.com/myapps/details?appId=<app_uuid> dashboard including an end slash.
Also - what is the body of the 400 error, there should be something like
{
"error": "invalid_grant"
}
Which will help you deduce what is wrong more easily.
https://developer.xero.com/documentation/oauth2/troubleshooting
I need to just a simple log in the webpage login page and how do I check that login is successful or not?
library(httr)
library(jsonlite)
library(tictoc)
library(data.table)
library(properties)
library(futile.logger)
library(crayon)
library(XML)
library(methods)
library(compare)
library(tictoc)
args = commandArgs(trailingOnly=TRUE)
server.name <- "lgloz050.lss.emc.com"
port.no <- "58443"
default.path <- "/APG/lookup/"
set_config(config(ssl_verifypeer = 0L))
config.s3 <- fread("Configuration_modify.csv")
config.s3$bc <- config.s3$testReport
config.s3$testReport <- gsub(">>","/", config.s3$testReport)
config.s3$testReport <- gsub(" ","%20", config.s3$testReport)
config.s3$link <- paste("https://",server.name,":",port.no,default.path,config.s3$testReport,"/report.csv", sep = "")
properties = read.csv2("Configuration.properties",sep = "=", blank.lines.skip = TRUE,header = FALSE,stringsAsFactors = FALSE )
colnames(properties) <- c("key", "value")
config.s3$link <- gsub("$","PH_", config.s3$link)
#config.s3$link
for(i in 1:nrow(properties)){
if(startsWith(properties[i,1],"$")){
print(properties[i,1])
for (j in 1: nrow(config.s3)) {
config.s3[j]$link = gsub(paste("PH_",substring(trimws(properties[i,1]),2),sep = "")
,trimws(properties[i,2]),config.s3[j]$link,ignore.case = TRUE)
}
}
}
result <- config.s3[, list(bc,TestCaseID,link),]
auth <- function(link,user.name="*****", password="******"){
res <- GET(link,add_headers("accept"="text/json"))
res <- POST('https://lgloz050.lss.emc.com:58443/APG/j_security_check'
,set_cookies=res$cookies
,body = "j_username=*****&j_password=******"
,add_headers("Content-Type" ="application/x-www-form-linkencoded" ))
return(res)
}
fetch <- function(link,save.location,cookies){
fetch.success = TRUE
res <- GET(link
,add_headers("Authorization"="Basic **************")
,set_cookies=cookies)
tryCatch({repot_data <- fread(content(res,"text"),header = TRUE);
fwrite(data.frame(repot_data),save.location,row.names = FALSE);
flog.info(green("'\u2713' - Fetch Completed successfully ..."))
flog.info(paste("link : ",link))},
error = function(e){fetch.success= FALSE; flog.error(paste("\u2715 - Not able to fetch data,file not created "))})
return(fetch.success)
}
config.s3$save.location = sub("TruthData","testData",config.s3$truthReport,ignore.case = T)
response = auth(config.s3[1]$link)
# Function Call - fetch all the report data
result[,fetch:=FALSE]
result[,fetch.time:=0]
pb <- winProgressBar(title="Fetching Reports... ", label="0%", min=0, max=100, initial=0,width = 500)
for (i in 1:nrow(config.s3)) {
tic()
getWinProgressBar(pb)
setWinProgressBar(pb, i*(100/nrow(config.s3)), label =paste(round(i*(100/nrow(config.s3)))," % \n",config.s3[i]$testReport))
flog.info(paste("report",i,"started",config.s3[i]$link))
fetch.success = fetch(config.s3[i]$link,config.s3[i]$save.location,response$cookies)
t <- toc()
t$toc
result[i]$fetch <- fetch.success
result[i]$fetch.time <- t$toc / 10000
}
close(pb)
result[,-c("link"),with=FALSE]
this is the code to fetch the CSV file but the file has HTML content of login page. please tell me where I am doing mistak and what i have to correct and modify to get the correct data.
suggest some procedure. Thanks in advance.
I got the solution which is as follows:
library(httr)
library(rvest)
url <- "https://lgloz050.lss.emc.com:58443/APG/"
dn_url <- "https://lgloz050.lss.emc.com:58443/APG/lookup/Report%20Library/Amazon%20S3/Inventory/Accounts/report.csv"
session <- html_session(url)
form <- html_form(session)[[1]]
fl_fm <- set_values(form,
j_username = "*****",
j_password = "********")
main_page <- submit_form(session, fl_fm)
downlaod <- jump_to(main_page,dn_url)
writeBin(downlaod$response$content, basename(dn_url))
on the execution of this code it will successfully log in and download the report and the downloaded report has the same content which is required. I do this for one file next I am trying to download the multiple files in one execution.
Thanks to you all for your support. let me know if there any other solution possible or any kind of modification is required in the above code.
I want to extract a table periodicaly from below site.
price list changes when clicked building block names(BLOK 16 A, BLOK 16 B, BLOK 16 C, ...) . URL doesn't change, page changes by trigering
javascript:__doPostBack('ctl00$ContentPlaceHolder1$DataList2$ctl04$lnk_blok','')
I've tried 3 ways after searching google and starckoverflow.
what I've tried no 1: this doesn't triger doPostBack event.
postForm( "http://www.kentkonut.com.tr/tr/modul/projeler/daire_fiyatlari.aspx?id=44", ctl00_ContentPlaceHolder1_DataList2_ctl03_lnk_blok="ctl00$ContentPlaceHolder1$DataList2$ctl03$lnk_blok")
what I've tried no 2: selenium remote seem to works on (http://localhost:4444/) but remotedriver doesn't navigate. returns this error. (Error in checkError(res) :
Undefined error in httr call. httr output: length(url) == 1 is not TRUE)
library(RSelenium)
startServer()
remDr <- remoteDriver()
remDr <- remoteDriver(remoteServerAddr = "localhost"
, port = 4444L, browserName = "firefox")
remDr$open()
remDr$getStatus()
remDr$navigate("http://www.kentkonut.com.tr/tr/modul/projeler/daire_fiyatlari.aspx?id=44")
what I've tried no 3: this another way to triger dopostback event. it doesn't navigate.
base.url <- "http://www.kentkonut.com.tr/tr/modul/projeler/",
event.target <- 'ctl00$ContentPlaceHolder1$DataList2$ctl03$lnk_blok',
action <- "daire_fiyatlari.aspx?id=44"
ftarget <- paste0(base.url, action)
dum <- getURL(ftarget)
event.val <- unlist(strsplit(dum,"__EVENTVALIDATION\" value=\""))[2]
event.val <- unlist(strsplit(event.val,"\" />\r\n\r\n<script"))[1]
view.state <- unlist(strsplit(dum,"id=\"__VIEWSTATE\" value=\""))[2]
view.state <- unlist(strsplit(view.state,"\" />\r\n\r\n\r\n<script"))[1]
web.data <- postForm(ftarget, "form name" = "ctl00_ContentPlaceHolder1_DataList2_ctl03_lnk_blok",
"method" = "POST",
"action" = action,
"id" = "ctl00_ContentPlaceHolder1_DataList2_ctl03_lnk_blok",
"__EVENTTARGET"=event.target,
"__EVENTVALIDATION"=event.val,
"__VIEWSTATE"=view.state)
thanks for your help.
library(rvest)
url<-"http://www.kentkonut.com.tr/tr/modul/projeler/daire_fiyatlari.aspx?id=44"
pgsession<-html_session(url)
t<-html_table(html_nodes(read_html(pgsession), css = "#ctl00_ContentPlaceHolder1_DataList1"), fill= TRUE)[[1]]
even_indices<-seq(2,length(t$X1),2)
t<-t[even_indices,]
t<-t[2:(length(t$X1)),]
EDITED CODE:
library(rvest)
url<-"http://www.kentkonut.com.tr/tr/modul/projeler/daire_fiyatlari.aspx?id=44"
pgsession<-html_session(url)
pgform<-html_form(pgsession)[[1]]
page<-rvest:::request_POST(pgsession,"http://www.kentkonut.com.tr/tr/modul/projeler/daire_fiyatlari.aspx?id=44",
body=list(
`__VIEWSTATE`=pgform$fields$`__VIEWSTATE`$value,
`__EVENTTARGET`="ctl00$ContentPlaceHolder1$DataList2$ctl01$lnk_blok",
`__EVENTARGUMENT`="",
`__VIEWSTATEGENERATOR`=pgform$fields$`__VIEWSTATEGENERATOR`$value,
`__VIEWSTATEENCRYPTED`=pgform$fields$`__VIEWSTATEENCRYPTED`$value,
`__EVENTVALIDATION`=pgform$fields$`__EVENTVALIDATION`$value
),
encode="form"
)
# in the above example change eventtarget as "ctl00$ContentPlaceHolder1$DataList2$ctl02$lnk_blok" to get different table
t<-html_table(html_nodes(read_html(page), css = "#ctl00_ContentPlaceHolder1_DataList1"), fill= TRUE)[[1]]
even_indices<-seq(2,length(t$X1),2)
t<-t[even_indices,]
t<-t[2:(length(t$X1)),]
I tried to pull my location data from openpaths.cc to use it with R.
The API uses OAuth and is documented here, however, it only provides an example in Python.
After looking around how to handle OAuth (which I am barely familiar with) in R, I found ROAuth, so I used the usage example provided as a basis.
According to the API-documentation, the endpoint for all requests is https://openpaths.cc/api/1, and I have my access key and access secret, so I naively plugged them in for cKey, cSecret, reqURL, accessURL, authURL, and testURL, but only got "bad request" as a result from the credentials$handshake() line.
reqURL <- "https://openpaths.cc/api/1"
accessURL <- "https://openpaths.cc/api/1"
authURL <- "https://openpaths.cc/api/1"
cKey <- "key"
cSecret <- "secret"
testURL <- "https://openpaths.cc/api/1"
credentials <- OAuthFactory$new(consumerKey=cKey,
consumerSecret=cSecret,
requestURL=reqURL,
accessURL=accessURL,
authURL=authURL,
needsVerifier=TRUE)
credentials$handshake()
## the GET isn’t strictly necessary as that’s the default
credentials$OAuthRequest(testURL, "GET")
While I feel like I have no idea what I'm doing, I at least verified that ROAuth is capable of using the HMAC-SHA1 method, wich is required by openpaths.
EDIT: I have ROAuth version 0.9.3 installed
EDIT2: After learning about httr, I thought this might be the appropriate library for the task, however I still could not produce any usable results, since the token creation via oauth1.0_token only lead to a Bad request again.
I think my primary problem is the lack of API documentation from openpaths.cc. With all these tools, I still have no idea how to properly use them.
Here is as far as I got. I receive a "400 Not Authorized", maybe this is due to the fact that my openpaths account is not connected to foursquare, maybe something is wrong with the code. Please try it out!
Required packages:
library(RCurl)
library(digest)
library(base64)
Some functions borrowed/adapted from ROAuth:
## Get a random sequence of characters.
## Nonce - number used only once.
genNonce <- function(len = 15L + sample(1:16, 1L)) {
els <- c(letters, LETTERS, 0:9, "_")
paste(sample(els, len, replace = TRUE), collapse = "")
}
## this function is derived from utils::URLencode
## Characters not in the unreserved character set ([RFC3986] section 2.3) MUST be encoded
## unreserved = ALPHA, DIGIT, '-', '.', '_', '~'
## cf. http://oauth.net/core/1.0/#encoding_parameters
encodeURI <- function(URI, ...) {
if (!is.character(URI)) {
URI
} else {
OK <- "[^-A-Za-z0-9_.~]"
x <- strsplit(URI, "")[[1L]]
z <- grep(OK, x)
if (length(z)) {
y <- sapply(x[z], function(x) paste("%", toupper(as.character(charToRaw(x))),
sep = "", collapse = ""))
x[z] <- y
}
paste(x, collapse = "")
}
}
## we escape the values of the parameters in a special way that escapes
## the resulting % prefix in the escaped characters, e.g. %20 becomes
## %2520 as %25 is the escape for %
## cf. http://tools.ietf.org/html/rfc5849#section-3.4.1.3.2
normalizeParams <- function(params, escapeFun) {
names(params) <- sapply(names(params), escapeFun, post.amp = TRUE)
params <- sapply(params, escapeFun, post.amp = TRUE)
## If two or more parameters share the same name, they are sorted by their value.
params <- params[order(names(params), params)]
return(paste(names(params), params, sep = "=", collapse = "&"))
}
## From Ozaki Toru's code at https://gist.github.com/586468
signWithHMAC <- function(key, data) {
blockSize <- 64
hashlength <- 20
innerpad <- rawToBits(as.raw(rep(0x36, blockSize)))
outerpad <- rawToBits(as.raw(rep(0x5C, blockSize)))
zero <- rep(0 ,64)
HexdigestToDigest <- function(digest) {
as.raw(strtoi(substring(digest, (1:hashlength)*2-1,
(1:hashlength)*2), base=16))
}
mac <- function(pad, text) {
HexdigestToDigest(digest(append(packBits(xor(key, pad)), text),
algo='sha1', serialize=FALSE))
}
if(nchar(key) >= 64) {
keyDigested <- digest(key, algo="sha1", serialize=FALSE)
key <- intToUtf8(strtoi(HexdigestToDigest(keyDigested), base=16))
}
key <- rawToBits(as.raw(append(utf8ToInt(key), zero)[1:blockSize]))
base64(mac(outerpad, mac(innerpad, charToRaw(data))))[1]
}
## Sign an request made up of the URL, the parameters as a named character
## vector the consumer key and secret and the token and token secret.
signRequest <- function(uri, consumerKey, consumerSecret, params=character(),
oauthKey = "", oauthSecret = "", httpMethod = "GET",
nonce = genNonce(),
timestamp = Sys.time()) {
httpMethod <- toupper(httpMethod)
params["oauth_nonce"] <- nonce
params["oauth_timestamp"] <- as.integer(timestamp)
params["oauth_consumer_key"] <- consumerKey
params["oauth_signature_method"] <- 'HMAC-SHA1'
params["oauth_version"] <- '1.0'
if(oauthKey != "") params["oauth_token"] <- oauthKey
odat <- paste(
encodeURI(httpMethod), encodeURI(uri),
encodeURI(normalizeParams(params, encodeURI), post.amp = TRUE),
sep = "&"
)
okey <- encodeURI(consumerSecret)
if(oauthSecret != "") okey <- paste(okey, encodeURI(oauthSecret), sep = "&")
params["oauth_signature"] <- signWithHMAC(okey, odat)
return(params)
}
Now this function tries to replicate the example at the openpaths website:
openpaths <- function(
access_key=getOption("openpaths.access_key"),
secret_key=getOption("openpaths.secret_key"),
curl=getCurlHandle()) {
uri <- 'https://openpaths.cc/api/1'
params <- signRequest(uri, consumerKey=access_key, consumerSecret=secret_key)
oa_header <- paste(names(params), params, sep="=", collapse=",")
ret <- getURL(
uri,
curl=curl,
.opts=list(
header=TRUE,
verbose=TRUE,
httpheader=c(Authorization=paste("OAuth ", oa_header, sep="")),
ssl.verifypeer = TRUE,
ssl.verifyhost = TRUE,
cainfo = system.file("CurlSSL", "cacert.pem", package = "RCurl")
)
)
return(ret)
}
I've made some progress on this problem, although it's challenging due
to the flakiness of the site, and the custom OAuth process that they're
using. First you'll need to install development version of httr - this
exports some previously internal functions.
devtools::install_github("hadley/httr")
OpenPaths is unusual in that the app secret and key are the same as the
token and token secret. This means we need to write a custom auth
header:
library(httr)
app <- oauth_app("OpenPaths", "JSLEKAPZIMFVFROHBDT4KNBVSI")
#> Using secret stored in environment variable OPENPATHS_CONSUMER_SECRET
# Implement custom header for 2-leg authentication, and oauth_body_hash
auth_header <- function(url, method = "GET") {
oauth_signature(url, method, app, app$key, app$secret,
# Use sha1 of empty string since http request body is empty
body_hash = "da39a3ee5e6b4b0d3255bfef95601890afd80709")
}
Then you can use this to sign your request. This is currently failing
for me because the site seems to be down (again).
url <- "https://openpaths.cc/api/1"
r <- GET(url, oauth_header(auth_header(url)))
stop_for_status(r)
content(r)