Working in R, but that shouldn't really matter.
I want to gather all tweets after : https://twitter.com/ChrisChristie/status/663046613779156996
So Tweet ID : 663046613779156996
base = "https://ontributor_details = "contributor_details=true"
## include_rts
include_rts = "include_rts=true"
## exclude_replies
exclude_replies = "exclude_replies=false"api.twitter.com/1.1/statuses/user_timeline.json?"
queryName = "chrischristie"
query = paste("q=", queryName, sep="")
secondary_url = paste(query, count, contributor_details,include_rts,exclude_replies, sep="&")
final_url = paste(base, secondary_url, sep="")
timeline = GET(final_url, sig)
This (the above) works. There is no since_id. The URL comes out to be
"https://api.twitter.com/1.1/statuses/user_timeline.json?q=chrischristie&count=200&contributor_details=true&include_rts=true&exclude_replies=false"
The below does not, just by adding in the following
cur_since_id_url = "since_id=663046613779156996"
secondary_url = paste(query, count,
contributor_details,include_rts,exclude_replies,cur_since_id_url, sep="&")
final_url = paste(base, secondary_url, sep="")
timeline = GET(final_url, sig)
The url for the above there is
"https://api.twitter.com/1.1/statuses/user_timeline.json?q=chrischristie&count=200&contributor_details=true&include_rts=true&exclude_replies=false&since_id=663046613779156992"
This seems to work:
require(httr)
myapp <- oauth_app(
"twitter",
key = "......",
secret = ".......")
twitter_token <- oauth1.0_token(oauth_endpoints("twitter"), myapp)
req <- GET("https://api.twitter.com/1.1/statuses/user_timeline.json",
query = list(
screen_name="chrischristie",
count=10,
contributor_details=TRUE,
include_rts=TRUE,
exclude_replies=FALSE,
since_id=663046613779156992),
config(token = twitter_token))
content(req)
Have a look at GET statuses/user_timeline
Related
I'm trying to upload some graphics from R, but I am getting 401 responses saying that I have an invalid signature. https://cloudinary.com/documentation/upload_images#uploading_with_a_direct_call_to_the_rest_api
library('httr')
library('digest')
domain = "test"
timestamp = round(as.numeric(Sys.time()))
to_sign = list(
public_id = paste0(domain,'_sales'),
timestamp = timestamp
)
secret = "SECRET"
almost_signature = paste0(
paste(names(to_sign),to_sign, sep="=", collapse = "&"),
secret)
signature = digest(almost_signature, algo="sha1", serialize = FALSE)
image = file.path(getwd(),paste0(domain,".png"))
values = list(
file = httr::upload_file(image),
api_key = "KEY",
timestamp = timestamp,
public_id = domain,
signature = signature
)
r <- POST("https://api.cloudinary.com/v1_1/hvgdpyed8/image/upload", body=values, content_type("multipart/form-data"))
Response
content(r)
$error
$error$message
[1] "Invalid Signature 6a5d9c05c11e9ea37bee8a717c2b9cdb75c34628. String to sign - 'public_id=test×tamp=1660676708'."
Testing Signature Generation
https://cloudinary.com/documentation/upload_images#generating_authentication_signatures
to_sign = list(eager="w_400,h_300,c_pad|w_260,h_200,c_crop",
public_id="sample_image",
timestamp = "1315060510")
secret = 'abcd'
almost_signature = paste0(
paste(names(to_sign),to_sign, sep="=", collapse = "&"),
secret)
almost_signature == "eager=w_400,h_300,c_pad|w_260,h_200,c_crop&public_id=sample_image×tamp=1315060510abcd"
correct_hash = "bfd09f95f331f558cbd1320e67aa8d488770583e"
sha1(almost_signature)
digest(almost_signature, algo="sha1", serialize = FALSE) == correct_hash
I use the UN Comtrade data API with R.
library(rjson)
get.Comtrade <- function(url="http://comtrade.un.org/api/get?"
,maxrec=50000
,type="C"
,freq="A"
,px="HS"
,ps="now"
,r
,p
,rg="all"
,cc="TOTAL"
,fmt="json"
)
{
string<- paste(url
,"max=",maxrec,"&" #maximum no. of records returned
,"type=",type,"&" #type of trade (c=commodities)
,"freq=",freq,"&" #frequency
,"px=",px,"&" #classification
,"ps=",ps,"&" #time period
,"r=",r,"&" #reporting area
,"p=",p,"&" #partner country
,"rg=",rg,"&" #trade flow
,"cc=",cc,"&" #classification code
,"fmt=",fmt #Format
,sep = ""
)
if(fmt == "csv") {
raw.data<- read.csv(string,header=TRUE)
return(list(validation=NULL, data=raw.data))
} else {
if(fmt == "json" ) {
raw.data<- fromJSON(file=string)
data<- raw.data$dataset
validation<- unlist(raw.data$validation, recursive=TRUE)
ndata<- NULL
if(length(data)> 0) {
var.names<- names(data[[1]])
data<- as.data.frame(t( sapply(data,rbind)))
ndata<- NULL
for(i in 1:ncol(data)){
data[sapply(data[,i],is.null),i]<- NA
ndata<- cbind(ndata, unlist(data[,i]))
}
ndata<- as.data.frame(ndata)
colnames(ndata)<- var.names
}
return(list(validation=validation,data =ndata))
}
}
}
However, sometimes it fails to connect server and I need to run the code several times to start working. Solution given here, to use Retry() function, which retries a request until it succeeds, seems attractive.
However, I have some difficulties implementing this function in the code given above. has anybody used it before and knows how to recode it?
An API call using httr::RETRY could look like the following:
library(httr)
library(jsonlite)
res <- RETRY(
verb = "GET",
url = "http://comtrade.un.org/",
path = "api/get",
encode = "json",
times = 3,
query = list(
max = 50000,
type = "C",
freq = "A",
px = "HS",
ps = "now",
r = 842,
p = "124,484",
rg = "all",
cc = "TOTAL",
fmt = "json"
)
)
# alternativ: returns dataset as a `list`:
# parsed_content <- content(res, as = "parsed")
# returns dataset as a `data.frame`:
json_content <- content(res, as = "text")
parsed_content <- parse_json(json_content, simplifyVector = TRUE)
parsed_content$validation
parsed_content$dataset
I'd suggest rewriting the get.Comtrade function using httr:
get.Comtrade <- function(verb = "GET",
url = "http://comtrade.un.org/",
path = "api/get",
encode = "json",
times = 3,
max = 50000,
type = "C",
freq = "A",
px = "HS",
ps = "now",
r,
p,
rg = "all",
cc = "TOTAL",
fmt = "json") {
res <- httr::RETRY(
verb = verb,
url = url,
path = path,
encode = encode,
times = times,
query = list(
max = max,
type = type,
freq = freq,
px = px,
ps = ps,
r = r,
p = p,
rg = rg,
cc = cc,
fmt = fmt
)
)
jsonlite::parse_json(content(res, as = "text"), simplifyVector = TRUE)
}
s1 <- get.Comtrade(r = "842", p = "124,484", times = 5)
print(s1)
Please see this and this for more information on library(httr).
Is there smooth solution to query data from Google Admob API into R environment? (for e.g. similar to googleAnalyticsR package).
So if anyone ever is looking for an answer the solution that i worked out is to use library(httr) and library(jsonlite) which are general packages for managing API's
First set up your google app for admob and generate oauth credentials,
store them in as described below and authorize token.
Obtain Token:
options(admob.client_id = client.id)
options(admob.client_secret = key.secret)
# GETTING OAUTH TOKEN
token = oauth2.0_token(endpoint = oauth_endpoints("google"), # 'google is standard
app = oauth_app(appname = "google",
key = getOption('admob.client_id'),
secret = getOption("admob.client_secret")
),
scope = r"{https://www.googleapis.com/auth/admob.report}",
use_oob = TRUE,
cache = TRUE)
You can generate your body request in within avaible google documentation in here:
https://developers.google.com/admob/api/v1/reference/rest/v1/accounts.networkReport/generate
I'am passing mine as.list:
`json body` = toJSON(list(`reportSpec` = list(
`dateRange` = list(
`startDate` = list(
`year` = 2021,
`month` = 10,
`day` = 20),
`endDate` = list(
`year` = 2021,
`month` = 10,
`day` = 22
)),
`dimensions` = list("DATE"),
`metrics` = list("IMPRESSIONS")
)), auto_unbox = TRUE)
Query your request:
test = POST(url = 'https://admob.googleapis.com/v1/accounts/YOURPUBIDGOESINHERE/networkReport:generate',
add_headers(`authorization` = paste("Bearer",
token$credentials$access_token)),
add_headers(`content-type` = "application/json"),
body = `json body`
)
Finalize with some data cleansing and you are done.
jsonText = content(test, as = 'text')
df = fromJSON(jsonText, flatten = T)
df = na.omit(df[c('row.metricValues.IMPRESSIONS.integerValue',
'row.dimensionValues.DATE.value')]) # select only needed columns
row.names(df) = NULL # reindex rows
df
Update:
Here is code that shows how to get an access token. I also use the test api here which is free (no credit card required).
The first api call to test.api.amadeus.com/v2/shopping/flight-offers is shown.
It is the second api call to test.api.amadeus.com/v1/shopping/flight-offers/pricing api that I don't know how to format.
My question remains, what is the correct way to call the second API using R?
R Script
library("tidyverse")
library("httr")
library("rjson")
amadeus_api_key_prod <- Sys.getenv("AMADEUS_API_KEY")
amadeus_api_secret_prod <- Sys.getenv("AMADEUS_SECRET")
# Initialize variables
tmp_origin <- NULL
tmp_dest <- NULL
tmp_avg_total_fare <- NULL
# Get Token
response <- POST("https://test.api.amadeus.com/v1/security/oauth2/token",
add_headers("Content-Type" = "application/x-www-form-urlencoded"),
body = list(
"grant_type" = "client_credentials",
"client_id" = amadeus_api_key_prod,
"client_secret" = amadeus_api_secret_prod),
encode = "form")
response
rsp_content <- content(response, as = "parsed", type = "application/json")
access_token <- paste0("Bearer ", rsp_content$access_token)
origin <- "JFK"
dest <- "LHR"
dep_date <- "2021-12-01"
return_date <- "2021-12-18"
max_num_flights <- 1
url <- paste0("https://test.api.amadeus.com/v2/shopping/flight-offers?originLocationCode=",
origin,
"&destinationLocationCode=",
dest,
"&departureDate=",
dep_date,
"&returnDate=",
return_date,
"&max=",
max_num_flights,
"&adults=1&nonStop=false&travelClass=ECONOMY&max=1¤cyCode=CAD")
# Get flight info
response <- GET(url,
add_headers("Content-Type" = "application/x-www-form-urlencoded",
"Authorization" = access_token),
encode = "form")
response
rsp_content <- content(response, as = "parsed", type = "application/json")
rsp_content
# Get current, more detailed flight info
# This is the part I do not know how to do
url2 <- "https://test.api.amadeus.com/v1/shopping/flight-offers/pricing"
flt_info <- toJSON(rsp_content[["data"]])
response2 <- GET(url2,
add_headers("Authorization" = access_token),
body = list(
"priceFlightOffersBody" = flt_info
),
encode = "form")
response2
rsp_content2 <- content(response2, as = "parsed", type = "application/json")
rsp_content2
Original Question
I am using the Amadeus flight info api to retrieve prices for flights.
My understanding is that to get complete price info requires two steps.
Call to https://api.amadeus.com/v2/shopping/flight-offers api
Call to https://api.amadeus.com/v1/shopping/flight-offers/pricing api
I can successfully perform the first api call with origin, destination, dates, etc.
The second call, confirms the pricing is still available and has a more detailed breakdown of the fare than does the first api call. It is this detailed breakdown that I am most interested in.
I am having trouble understanding what info that gets returned from the first api call needs to be passed to the second api call and in what format.
Below, I have included the data structure that gets returned from the first api call and my failed attempt to call the second api.
What is the correct way to call the second API using R?
Links to what I believe is the relevant documentation:
https://developers.amadeus.com/self-service/category/air/api-doc/flight-offers-price/api-reference
https://github.com/amadeus4dev/amadeus-code-examples/blob/master/flight_offers_price/v1/post/curl/flight_offers_price.sh
# Data structure returned from call to https://api.amadeus.com/v2/shopping/flight-offers
# For YYZ to YOW return, Dec 1-18 economy
rsp_content <- list(meta = list(count = 1L, links = list(self = "https://api.amadeus.com/v2/shopping/flight-offers?originLocationCode=YYZ&destinationLocationCode=YOW&departureDate=2021-12-01&returnDate=2021-12-18&max=1&adults=1&nonStop=false&travelClass=ECONOMY&max=1¤cyCode=CAD")),
data = list(list(type = "flight-offer", id = "1", source = "GDS",
instantTicketingRequired = FALSE, nonHomogeneous = FALSE,
oneWay = FALSE, lastTicketingDate = "2021-08-07", numberOfBookableSeats = 7L,
itineraries = list(list(duration = "PT1H10M", segments = list(
list(departure = list(iataCode = "YYZ", terminal = "3",
at = "2021-12-01T11:00:00"), arrival = list(iataCode = "YOW",
at = "2021-12-01T12:10:00"), carrierCode = "WS",
number = "3462", aircraft = list(code = "DH4"),
duration = "PT1H10M", id = "1", numberOfStops = 0L,
blacklistedInEU = FALSE))), list(duration = "PT1H21M",
segments = list(list(departure = list(iataCode = "YOW",
at = "2021-12-18T10:45:00"), arrival = list(iataCode = "YYZ",
terminal = "3", at = "2021-12-18T12:06:00"),
carrierCode = "WS", number = "3463", aircraft = list(
code = "DH4"), duration = "PT1H21M", id = "2",
numberOfStops = 0L, blacklistedInEU = FALSE)))),
price = list(currency = "CAD", total = "232.78", base = "125.00",
fees = list(list(amount = "0.00", type = "SUPPLIER"),
list(amount = "0.00", type = "TICKETING")), grandTotal = "232.78"),
pricingOptions = list(fareType = list("PUBLISHED"), includedCheckedBagsOnly = FALSE),
validatingAirlineCodes = list("WS"), travelerPricings = list(
list(travelerId = "1", fareOption = "STANDARD", travelerType = "ADULT",
price = list(currency = "CAD", total = "232.78",
base = "125.00"), fareDetailsBySegment = list(
list(segmentId = "1", cabin = "ECONOMY", fareBasis = "LAVD0TBJ",
brandedFare = "BASIC", class = "E", includedCheckedBags = list(
quantity = 0L)), list(segmentId = "2",
cabin = "ECONOMY", fareBasis = "LAVD0ZBI",
brandedFare = "BASIC", class = "E", includedCheckedBags = list(
quantity = 0L))))))), dictionaries = list(
locations = list(YOW = list(cityCode = "YOW", countryCode = "CA"),
YYZ = list(cityCode = "YTO", countryCode = "CA")),
aircraft = list(DH4 = "DE HAVILLAND DHC-8 400 SERIES"),
currencies = list(CAD = "CANADIAN DOLLAR"), carriers = list(
WS = "WESTJET")))
# Get full pricing info
url2 <- "https://api.amadeus.com/v1/shopping/flight-offers/pricing"
# Get pricing info
response2 <- GET(url2,
add_headers("Authorization" = access_token),
body = list(
"priceFlightOffersBody" = rsp_content[["data"]][[1]]
),
encode = "form")
response2
rsp_content2 <- content(response2, as = "parsed", type = "application/json")
rsp_content2
You can take a look at this blog article, it explains how the data needs to be passed between the 3 endpoints and it has a video showing it on Postman.
You can check some of the code samples the Amadeus for Developers team has built. Here for Flight Offers Price (in different programming languages but not R) and here for Flight Create Orders (that includes the previous steps of search and price).
They have a couple of demo applications as well that show you how to combine these endpoints to build a flight booking engine, one of them in Python that you can find here.
It seems to work with the following, for the second request.
Changes :
Use POST, not GET
While the documentation names the body priceFlightOffersBody, it should actually be just the data part, no need to encapsulate in another list.
The data part is not exactly the same as the result of the first request.
Pass the data part as an R list and use POST argument encode = "json".
url2 <- "https://test.api.amadeus.com/v1/shopping/flight-offers/pricing"
flt_info <- list(data = list(type = "flight-offers-pricing",
flightOffers = rsp_content$data))
response2 <- POST(url2,
add_headers(Authorization = access_token),
body = flt_info,
encode = "json")
rsp_content2 <- content(response2, as = "parsed", type = "application/json")
rsp_content2
I am scraping some data from an API, and my code works just fine as long as I extract pages 0 to 98. Whenever my loop reaches 99, I get an error Error: Internal Server Error (HTTP 500)..
Tried to find an answer but I am only proficient in R and C# and cannot understand Python or other.
keywords = c('ABC OR DEF')
parameters <- list(
'q' = keywords,
num_days = 1,
language = 'en',
num_results = 100,
page = 0,
'api_key' = '123456'
)
response <- httr::GET(get_url, query = parameters)
# latest_page_number <- get_last_page(parsed)
httr::stop_for_status(response)
content <- httr::content(response, type = 'text', encoding = 'utf-8')
parsed <- jsonlite::fromJSON(content, simplifyVector = FALSE, simplifyDataFrame = TRUE)
num_pages = round(parsed[["total_results"]]/100)
print(num_pages)
result = parsed$results
for(x in 1:(num_pages))
{
print(x)
parameters <- list(
'q' = keywords,
page = x,
num_days = 7,
language = 'en',
num_results = 100,
'api_key' = '123456'
)
response <- httr::GET(get_url, query = parameters)
httr::stop_for_status(response)
content <- httr::content(response, type = 'text', encoding = 'utf-8')
# content <- httr::content(response)
parsed <- jsonlite::fromJSON(content, simplifyVector = FALSE, simplifyDataFrame = TRUE)
Sys.sleep(0.2)
result = rbind(result,parsed$results[,colnames(result)])
}