I am trying to use the recently released AWS Lambda HTTPS endpoints feature: https://aws.amazon.com/blogs/aws/announcing-aws-lambda-function-urls-built-in-https-endpoints-for-single-function-microservices
I have created an AWS Lambda function called multiply that expects an x and a y in the request body and returns the product. I created the HTTPS endpoint and set it to IAM authentication. Now, I'd like to call it from R with help of the aws.signature package.
The expected result is an HTTP response containing the Lambda function's output.
But I run into error "400 Bad Request". I suspect that the action parameter for aws.signature::signature_v4_auth must be set, but I can't figure out what it should be.
library(httr)
library(jsonlite)
function_url <- "https://<url-id>.lambda-url.eu-central-1.on.aws"
region <- "eu-central-1"
request_body <- toJSON(list(x = 2, y = 4), auto_unbox = TRUE)
headers <- list()
headers[["x-amz-date"]] <- format(Sys.time(), "%Y%m%dT%H%M%SZ", tz = "UTC")
headers[["host"]] <- function_url
headers[["content-type"]] <- "application/json"
signature <- aws.signature::signature_v4_auth(
datetime = headers[["x-amz-date"]],
region = region,
action = "",
service = "lambda",
verb = "POST",
canonical_headers = headers,
request_body = request_body,
key = "***",
secret = "***"
)
headers[["Authorization"]] <- signature[["SignatureHeader"]]
POST(
url = function_url,
body = request_body,
do.call(add_headers, headers)
)
There are two errors in the request:
The host should not include "https://". So the correct host is <url-id>.lambda-url.eu-central-1.on.aws
An empty action is not permitted. Use "/" instead.
I am running the following code which is taken from https://developer.twitter.com/en/docs/tutorials/getting-started-with-r-and-v2-of-the-twitter-ap with some slight changes because it is doesn't seem to work.
library(rjson)
#https://developer.twitter.com/en/docs/tutorials/getting-started-with-r-and-v2-of-the-twitter-ap
Sys.setenv(BEARER_TOKEN="{mybearertoken}")
require(httr)
require(jsonlite)
require(dplyr)
bearer_token <- Sys.getenv("TWITTER_BEARER")
headers <- c(`Authorization` = sprintf('Bearer %s', bearer_token))
params <- list(`user.fields` = 'description',
`expansion` = 'pinned_tweet_id')
handle <- "codewryte"
url_handle <- paste("https://api.twitter.com/2/tweets/", handle)
#url_handle <-"https://twitter.com/TwitterDev/status/1228393702244134912"
response <-
httr::GET(url = url_handle,
httr::add_headers(.headers = headers),
query = params)
obj <- httr::content(response, as = "text")
x <- fromJSON(obj)
I get the following error:
$errors[[2]]$message
[1] "The query parameter [expansion] is not one of [usernames,expansions,tweet.fields,user.fields]"
$title
[1] "Invalid Request"
$detail
[1] "One or more parameters to your request was invalid."
$type
[1] "https://api.twitter.com/2/problems/invalid-request"
I also tried https://api.twitter.com/2/users/by/username/codewryte which is my user handle with the same message.
Does anyone understand what this message means and how I can fix it?
The problem was I was sending a list where a string needed to be sent.
response <-httr::GET(url = url_handle,
httr::add_headers(.headers = headers),
query="expansions=pinned_tweet_id&user.fields=created_at&tweet.fields=created_at")
I am not sure why the tutorial had this list.
This is the step after the question that can be found here
I've run the following code:
pack <- c('curl','xml2','XML', 'plyr', 'dplyr','tidyr', 'httr', 'tools', 'lubridate',
'jsonlite', 'stringr', 'data.table', 'anytime', 'RCurl', 'rvest', 'opnessl', 'jose')
sapply(pack, function(x){
if(!require(x,character.only = T, quietly = T)) {install.packages(x, quiet = T)}
require(x, quietly = T, character.only = T)
})
#New Xero & WFM Api OAuth 2.0 credentials
Client_ID <- 'Your_Client_ID '
Client_secret<- 'Your_Client_secret'
XTID_Xero <- 'x30rscript'#Referral_ID
Redirect_URI <- 'https://developer.xero.com/' #OAuth 2.0 redirect URI
# Create the app
app <- oauth_app("RScript",
key = Client_ID,
secret = Client_secret,
redirect_uri = Redirect_URI
)
# Create the endpoint
create_endpoint <- function()
{
request <- "https://identity.xero.com/connect/token"
authorize <- "https://login.xero.com/identity/connect/authorize"
access <- "https://identity.xero.com/connect/token"
httr::oauth_endpoint(request, authorize, access)
}
api <- create_endpoint()
header <- httr::add_headers(Authorization=paste0("Basic ", RCurl::base64Encode(charToRaw(paste0(Client_ID, ":", Client_secret)))))
content_type <- httr::content_type("application/x-www-form-urlencoded")
# Define the scope - https://developer.xero.com/documentation/oauth2/scopes
scope_WFM <- "openid profile offline_access payroll.employees.read payroll.payruns.read payroll.payslip.read payroll.timesheets.read accounting.transactions.read accounting.reports.read accounting.journals.read"
# Get the code
httr::BROWSE(oauth2.0_authorize_url(api, app, scope = scope_WFM))
#get the code from the URL displayed in your browser
code_xero <- 'code_xero'
state_xero <- 'state_xero'
token <- httr::oauth2.0_token(
endpoint = api,
app = app,
scope = scope_WFM,
config_init = c(header, content_type),
use_basic_auth = TRUE,
query_authorize_extra = list(prompt = "login"),
type = "code",
credentials = oauth2.0_access_token(api, app, code_xero),
cache = FALSE
)
#get your xero-tenant-id
access <- GET("https://api.xero.com/connections", config = token)
connections <- content(access, 'text')
connections <- fromJSON(connections, flatten = T)
connections
# This is where I get an error....
GET("https://api.xero.com/api.xro/2.0/banktransactions",
config = token,
authenticate(Client_ID, Client_secret))
This returns the following:
Response [https://api.xero.com/api.xro/2.0/banktransactions]
Date: 2020-12-01 10:59
Status: 403
Content-Type: application/json
Size: 150 B
I'm trying to access Xero to pull out the current cash position, Debtors Days, Turnover/Profit/Growth and I'm struggling to a response from any of the areas I need in R.
Any help would be much appreciated.
I was actually searching too see if anyone else is trying to use R with Xero when this popped up.
So in case anyone else needs this, the solution is to pass the token with the headers. Using httr::add_headers this would be like so:
httr::GET("https://api.xero.com/api.xro/2.0/Invoices", config = token, httr::add_headers(`Xero-tenant-id` = tenant_id))
I'm trying to scrape several comments on YouTube but my id doesn't work and I don't know if i have to use the API or the client id/secret:
>id <- "XXX"
>api <- "xxx"
>client <-"xxx"
>secret <- "xxx"
yt_oauth(client, secret)
yt.oauth(client, secret)
yt.ouath(client,secret, token = ' ')
yt_oauth(user, api, token = '')
# A<- yt_oauth(user,api)
install.packages("devtools")
library (devtools)
devtools::install_github("soodoku/tuber", build_vignettes = TRUE)
#quick overview of some important functions in tuber, see the vignette:
vignette("tuber-ex", package="tuber")
#Get All the Comments Including Replies
get_all_comments(video_id = "a-UQz7fqR3w")
yt.oauth(client, secret)
Error in yt.oauth(client, secret) : not found the function "yt.oauth"
yt_oauth(user, api, token = '')
Waiting for authentication in browser...Press Esc/Ctrl + C to abort
But it doesn't recognize my credentials
Thank you so much!!
I do not know if this is interesting for you, but I have been able to get the comments of a Youtube video with the following code :
library(rvest)
library(RSelenium)
port <- as.integer(4444)
rd <- rsDriver(chromever = "105.0.5195.52", browser = "chrome", port = port)
remDr <- rd$client
remDr$open()
url <- 'https://www.youtube.com/watch?v=0WzPPRUTQZ4'
remDr$navigate(url)
# Scroll down the page to load all the comments
for(i in 1 : 200)
{
print(i)
java_Script <- paste0("scroll(0,", i * 500, ")")
remDr$executeScript(java_Script)
}
web_Obj_Comments <- remDr$findElements("xpath", '//*[#id="content"]')
list_Comments <- list()
nb_Comments <- length(web_Obj_Comments) - 2
for(i in 1 : nb_Comments)
{
print(i)
list_Comments[[i]] <- web_Obj_Comments[[i + 1]]$getElementText()[[1]]
}
vector_Comments <- unlist(list_Comments)
vector_Comments <- vector_Comments[vector_Comments != ""]
I tried to pull my location data from openpaths.cc to use it with R.
The API uses OAuth and is documented here, however, it only provides an example in Python.
After looking around how to handle OAuth (which I am barely familiar with) in R, I found ROAuth, so I used the usage example provided as a basis.
According to the API-documentation, the endpoint for all requests is https://openpaths.cc/api/1, and I have my access key and access secret, so I naively plugged them in for cKey, cSecret, reqURL, accessURL, authURL, and testURL, but only got "bad request" as a result from the credentials$handshake() line.
reqURL <- "https://openpaths.cc/api/1"
accessURL <- "https://openpaths.cc/api/1"
authURL <- "https://openpaths.cc/api/1"
cKey <- "key"
cSecret <- "secret"
testURL <- "https://openpaths.cc/api/1"
credentials <- OAuthFactory$new(consumerKey=cKey,
consumerSecret=cSecret,
requestURL=reqURL,
accessURL=accessURL,
authURL=authURL,
needsVerifier=TRUE)
credentials$handshake()
## the GET isn’t strictly necessary as that’s the default
credentials$OAuthRequest(testURL, "GET")
While I feel like I have no idea what I'm doing, I at least verified that ROAuth is capable of using the HMAC-SHA1 method, wich is required by openpaths.
EDIT: I have ROAuth version 0.9.3 installed
EDIT2: After learning about httr, I thought this might be the appropriate library for the task, however I still could not produce any usable results, since the token creation via oauth1.0_token only lead to a Bad request again.
I think my primary problem is the lack of API documentation from openpaths.cc. With all these tools, I still have no idea how to properly use them.
Here is as far as I got. I receive a "400 Not Authorized", maybe this is due to the fact that my openpaths account is not connected to foursquare, maybe something is wrong with the code. Please try it out!
Required packages:
library(RCurl)
library(digest)
library(base64)
Some functions borrowed/adapted from ROAuth:
## Get a random sequence of characters.
## Nonce - number used only once.
genNonce <- function(len = 15L + sample(1:16, 1L)) {
els <- c(letters, LETTERS, 0:9, "_")
paste(sample(els, len, replace = TRUE), collapse = "")
}
## this function is derived from utils::URLencode
## Characters not in the unreserved character set ([RFC3986] section 2.3) MUST be encoded
## unreserved = ALPHA, DIGIT, '-', '.', '_', '~'
## cf. http://oauth.net/core/1.0/#encoding_parameters
encodeURI <- function(URI, ...) {
if (!is.character(URI)) {
URI
} else {
OK <- "[^-A-Za-z0-9_.~]"
x <- strsplit(URI, "")[[1L]]
z <- grep(OK, x)
if (length(z)) {
y <- sapply(x[z], function(x) paste("%", toupper(as.character(charToRaw(x))),
sep = "", collapse = ""))
x[z] <- y
}
paste(x, collapse = "")
}
}
## we escape the values of the parameters in a special way that escapes
## the resulting % prefix in the escaped characters, e.g. %20 becomes
## %2520 as %25 is the escape for %
## cf. http://tools.ietf.org/html/rfc5849#section-3.4.1.3.2
normalizeParams <- function(params, escapeFun) {
names(params) <- sapply(names(params), escapeFun, post.amp = TRUE)
params <- sapply(params, escapeFun, post.amp = TRUE)
## If two or more parameters share the same name, they are sorted by their value.
params <- params[order(names(params), params)]
return(paste(names(params), params, sep = "=", collapse = "&"))
}
## From Ozaki Toru's code at https://gist.github.com/586468
signWithHMAC <- function(key, data) {
blockSize <- 64
hashlength <- 20
innerpad <- rawToBits(as.raw(rep(0x36, blockSize)))
outerpad <- rawToBits(as.raw(rep(0x5C, blockSize)))
zero <- rep(0 ,64)
HexdigestToDigest <- function(digest) {
as.raw(strtoi(substring(digest, (1:hashlength)*2-1,
(1:hashlength)*2), base=16))
}
mac <- function(pad, text) {
HexdigestToDigest(digest(append(packBits(xor(key, pad)), text),
algo='sha1', serialize=FALSE))
}
if(nchar(key) >= 64) {
keyDigested <- digest(key, algo="sha1", serialize=FALSE)
key <- intToUtf8(strtoi(HexdigestToDigest(keyDigested), base=16))
}
key <- rawToBits(as.raw(append(utf8ToInt(key), zero)[1:blockSize]))
base64(mac(outerpad, mac(innerpad, charToRaw(data))))[1]
}
## Sign an request made up of the URL, the parameters as a named character
## vector the consumer key and secret and the token and token secret.
signRequest <- function(uri, consumerKey, consumerSecret, params=character(),
oauthKey = "", oauthSecret = "", httpMethod = "GET",
nonce = genNonce(),
timestamp = Sys.time()) {
httpMethod <- toupper(httpMethod)
params["oauth_nonce"] <- nonce
params["oauth_timestamp"] <- as.integer(timestamp)
params["oauth_consumer_key"] <- consumerKey
params["oauth_signature_method"] <- 'HMAC-SHA1'
params["oauth_version"] <- '1.0'
if(oauthKey != "") params["oauth_token"] <- oauthKey
odat <- paste(
encodeURI(httpMethod), encodeURI(uri),
encodeURI(normalizeParams(params, encodeURI), post.amp = TRUE),
sep = "&"
)
okey <- encodeURI(consumerSecret)
if(oauthSecret != "") okey <- paste(okey, encodeURI(oauthSecret), sep = "&")
params["oauth_signature"] <- signWithHMAC(okey, odat)
return(params)
}
Now this function tries to replicate the example at the openpaths website:
openpaths <- function(
access_key=getOption("openpaths.access_key"),
secret_key=getOption("openpaths.secret_key"),
curl=getCurlHandle()) {
uri <- 'https://openpaths.cc/api/1'
params <- signRequest(uri, consumerKey=access_key, consumerSecret=secret_key)
oa_header <- paste(names(params), params, sep="=", collapse=",")
ret <- getURL(
uri,
curl=curl,
.opts=list(
header=TRUE,
verbose=TRUE,
httpheader=c(Authorization=paste("OAuth ", oa_header, sep="")),
ssl.verifypeer = TRUE,
ssl.verifyhost = TRUE,
cainfo = system.file("CurlSSL", "cacert.pem", package = "RCurl")
)
)
return(ret)
}
I've made some progress on this problem, although it's challenging due
to the flakiness of the site, and the custom OAuth process that they're
using. First you'll need to install development version of httr - this
exports some previously internal functions.
devtools::install_github("hadley/httr")
OpenPaths is unusual in that the app secret and key are the same as the
token and token secret. This means we need to write a custom auth
header:
library(httr)
app <- oauth_app("OpenPaths", "JSLEKAPZIMFVFROHBDT4KNBVSI")
#> Using secret stored in environment variable OPENPATHS_CONSUMER_SECRET
# Implement custom header for 2-leg authentication, and oauth_body_hash
auth_header <- function(url, method = "GET") {
oauth_signature(url, method, app, app$key, app$secret,
# Use sha1 of empty string since http request body is empty
body_hash = "da39a3ee5e6b4b0d3255bfef95601890afd80709")
}
Then you can use this to sign your request. This is currently failing
for me because the site seems to be down (again).
url <- "https://openpaths.cc/api/1"
r <- GET(url, oauth_header(auth_header(url)))
stop_for_status(r)
content(r)