I used to code like this :
inets:start(),
ssl:start(),
ParaUrl = io_lib:format("http://xxx/passport?accessn=~s",[Access]),
ParaStr = io_lib:format("id=~p",[Id]),
RegUsr = httpc:request(post,{ParaUrl, [],"application/x-www-form-urlencoded", list_to_binary(ParaStr)},[],[])
but it's just for string not binary file, can anyone tell me how to post binary file with httpc:request?
It works for me:
{ok, F} = file:read_file("warning.gif"),
httpc:request(post,{Url, [],"multipart/form-data", F},[],[]).
I recieved on cowboy web server with function cowboy_req:body(Req):
{ok,<<71,73,70,56,57,97,11,0,11,0,230,64,0............>>,
{http_req,#Port<0.65562>,ranch_tcp,keepalive,<0.8736.0>,<<"POST">>,
'HTTP/1.1',
{{127,0,0,1},58154},
<<"localhost">>,undefined,8000,<<"/test1">>,
[<<"test1">>],
<<>>,undefined,[],
[{<<"content-type">>,<<"multipart/form-data">>},
{<<"content-length">>,<<"516">>},
{<<"te">>,<<>>},
{<<"host">>,<<"localhost:8000">>},
{<<"connection">>,<<"keep-alive">>}],
[{<<"content-length">>,516},
{<<"expect">>,undefined},
{<<"content-length">>,516},
{<<"connection">>,[<<"keep-alive">>]}],undefined,[],done,undefined,<<>>,true,waiting,[],<<>>,
undefined}}
Inspired by ActiveState recipe 146306 (and PYTHON HTTP POST BINARY FILE UPLOAD WITH PYCURL (PYTHON RECIPE)), altered by Formatting Multipart Formdata in Erlang. Thanks!
format_multipart_formdata(Boundary, Fields, Files) ->
FieldParts = lists:map(fun({FieldName, FieldContent}) ->
[lists:concat(["--", Boundary]),
lists:concat(["Content-Disposition: form-data; name=\"",atom_to_list(FieldName),"\""]),
"", FieldContent]
end, Fields),
FieldParts2 = lists:append(FieldParts),
FileParts = lists:map(fun({FieldName, FileName, FileContent}) ->
[lists:concat(["--", Boundary]),
lists:concat(["Content-Disposition: form-data; name=\"",atom_to_list(FieldName),"\"; filename=\"",FileName,"\""]),
lists:concat(["Content-Type: ", "application/octet-stream"]), "", FileContent]
end, Files),
FileParts2 = lists:append(FileParts),
EndingParts = [lists:concat(["--", Boundary, "--"]), ""],
Parts = lists:append([FieldParts2, FileParts2, EndingParts]),
string:join(Parts, "\r\n").
Usage:
{ok,BinStream} = file:read_file("./images/avatar.png"),
Data = binary_to_list(BinStream),
Boundary = "------WebKitFormBoundaryUscTgwn7KiuepIr1",
ReqBody = format_multipart_formdata(Boundary, [{uid,"123"}], [{avatar, "avatar", Data}]),
ContentType = lists:concat(["multipart/form-data; boundary=", Boundary]),
ReqHeader = [{"Content-Length", integer_to_list(length(ReqBody))}],
inets:start(),
ParaUrl = string:join(["http://www.example.com/avatar?access_token=",binary_to_list(token)],""),
RegUsr = httpc:request(post,{ParaUrl, ReqHeader,ContentType, ReqBody},[],[])
Related
I just started studying Lua.
Every time I request, I want to check the name parameter in the request parameter. However, it is actually found that the self.name has changed occasionally.
For example,
request A with params: request_id = 123 & name = ABC,
request B with params: request_id = 321 & name = EFG,
in the log, I found that there are requests_id = 123, but name = EFG.
Why is that? Is my class incorrectly written?
Here is the sample codeļ¼
main.lua:
local checker = require "checker"
local ch = checker:new()
if ch:check_name() then
ngx.header["Content-Type"] = "application/json"
ngx.status = ngx.HTTP_FORBIDDEN
ngx.exit(ngx.HTTP_FORBIDDEN)
end
checker.lua:
local utils = require "utils"
local _M = {}
function _M:new()
local o = {}
setmetatable(o, self)
self.__index = self
self.args = utils.get_req_args() or {}
local name = self.args["name"] or ""
local request_id = self.args["request_id"] or ""
self.name = name
return o
end
function _M:check_name()
ngx.log(ngx.ERR, "request_id: ", self.request_id, " name: ", self.name)
-- do some check ...
end
utils.lua
local json = require "cjson"
local _M = {}
function _M.new(self)
return self
end
function _M.get_req_args()
-- GET
local args = nil
if ngx.var.request_method == "GET" then
args = ngx.req.get_uri_args()
-- POST
elseif ngx.var.request_method == "POST" then
ngx.req.read_body()
local data = ngx.req.get_body_data()
args = json.decode(data)
end
return args
end
I have been trying to download all the YouTube comments on popular videos using python requests, but it has been throwing up the following error after about a quarter of the total comments:
{'error': {'code': 400, 'message': "The API server failed to successfully process the request. While this can be a transient error, it usually indicates that the request's input is invalid. Check the structure of the commentThread resource in the request body to ensure that it is valid.", 'errors': [{'message': "The API server failed to successfully process the request. While this can be a transient error, it usually indicates that the request's input is invalid. Check the structure of the commentThread resource in the request body to ensure that it is valid.", 'domain': 'youtube.commentThread', 'reason': 'processingFailure', 'location': 'body', 'locationType': 'other'}]}}
I found this thread detailing the same issue, and it seems that it is not possible to download all the comments on popular videos.
This is my code:
import argparse
import urllib
import requests
import json
import time
start_time = time.time()
class YouTubeApi():
YOUTUBE_COMMENTS_URL = 'https://www.googleapis.com/youtube/v3/commentThreads'
comment_counter = 0
with open("API_keys.txt", "r") as f:
key_list = f.readlines()
key_list = [key.strip('/n') for key in key_list]
def format_comments(self, results, likes_required):
comments_list = []
try:
for item in results["items"]:
comment = item["snippet"]["topLevelComment"]
likes = comment["snippet"]["likeCount"]
if likes < likes_required:
continue
author = comment["snippet"]["authorDisplayName"]
text = comment["snippet"]["textDisplay"]
str = "Comment by {}:\n \"{}\"\n\n".format(author, text)
str = str.encode('ascii', 'replace').decode()
comments_list.append(str)
self.comment_counter += 1
print("Comments downloaded:", self.comment_counter, end="\r")
except(KeyError):
print(results)
return comments_list
def get_video_comments(self, video_id, likes_required):
with open("API_keys.txt", "r") as f:
key_list = f.readlines()
key_list = [key.strip('/n') for key in key_list]
if self.comment_counter <= 900000:
key = self.key_list[0]
elif self.comment_counter <= 1800000:
key = self.key_list[1]
elif self.comment_counter <= 2700000:
key = self.key_list[2]
elif self.comment_counter <= 3600000:
key = self.key_list[3]
elif self.comment_counter <= 4500000:
key = self.key_list[4]
params = {
'part': 'snippet,replies',
'maxResults': 100,
'videoId': video_id,
'textFormat': 'plainText',
'key': key
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}
try:
#data = self.openURL(self.YOUTUBE_COMMENTS_URL, params)
comments_data = requests.get(self.YOUTUBE_COMMENTS_URL, params=params, headers=headers)
except ChunkedEncodingError:
tries = 5
print("Chunked Error. Retrying...")
for n in range(tries):
try:
x = 0
x += 1
print("Trying", x, "times")
response = session.post("https://www.youtube.com/comment_service_ajax", params=params, data=data, headers=headers)
comments_data = json.loads(response.text)
except ChunkedEncodingError as c:
print(c)
results = comments_data.json()
nextPageToken = results.get("nextPageToken")
commments_list = []
commments_list += self.format_comments(results, likes_required)
while nextPageToken:
params.update({'pageToken': nextPageToken})
try:
comments_data = requests.get(self.YOUTUBE_COMMENTS_URL, params=params, headers=headers)
except ChunkedEncodingError as c:
tries = 5
print("Chunked Error. Retrying...")
for n in range(tries):
try:
x = 0
x += 1
print("Trying", x, "times")
response = session.post("https://www.youtube.com/comment_service_ajax", params=params, data=data, headers=headers)
comments_data = json.loads(response.text)
except ChunkedEncodingError as c:
print(c)
results = comments_data.json()
nextPageToken = results.get("nextPageToken")
commments_list += self.format_comments(results, likes_required)
return commments_list
def get_video_id_list(self, filename):
try:
with open(filename, 'r') as file:
URL_list = file.readlines()
except FileNotFoundError:
exit("File \"" + filename + "\" not found")
list = []
for url in URL_list:
if url == "\n": # ignore empty lines
continue
if url[-1] == '\n': # delete '\n' at the end of line
url = url[:-1]
if url.find('='): # get id
id = url[url.find('=') + 1:]
list.append(id)
else:
print("Wrong URL")
return list
def main():
yt = YouTubeApi()
parser = argparse.ArgumentParser(add_help=False, description=("Download youtube comments from many videos into txt file"))
required = parser.add_argument_group("required arguments")
optional = parser.add_argument_group("optional arguments")
here: https://console.developers.google.com/apis/credentials")
optional.add_argument("--likes", '-l', help="The amount of likes a comment needs to be saved", type=int)
optional.add_argument("--input", '-i', help="URL list file name")
optional.add_argument("--output", '-o', help="Output file name")
optional.add_argument("--help", '-h', help="Help", action='help')
args = parser.parse_args()
# --------------------------------------------------------------------- #
likes = 0
if args.likes:
likes = args.likes
input_file = "URL_list.txt"
if args.input:
input_file = args.input
output_file = "Comments.txt"
if args.output:
output_file = args.output
list = yt.get_video_id_list(input_file)
if not list:
exit("No URLs in input file")
try:
vid_counter = 0
with open(output_file, "a") as f:
for video_id in list:
vid_counter += 1
print("Downloading comments for video ", vid_counter, ", id: ", video_id, sep='')
comments = yt.get_video_comments(video_id, likes)
if comments:
for comment in comments:
f.write(comment)
print('\nDone!')
except KeyboardInterrupt:
exit("User Aborted the Operation")
# --------------------------------------------------------------------- #
if __name__ == '__main__':
main()
The next best method would be to randomly sample them. Does anyone know if this is possible with the API V3?
Even if the API returns a processingFailure error, you could still catch that (or any other API error for that matter) for to terminate gracefully your pagination loop. This way your script will provide the top-level comments that it fetched from of the API prior to the occurrence of the first API error.
The error response provided by the YouTube Data API is (usually) of the following form:
{
"error": {
"errors": [
{
"domain": <string>,
"reason": <string>,
"message": <string>,
"locationType": <string>,
"location": <string>
}
],
"code": <integer>,
"message": <string>
}
}
Hence, you could have defined the following function:
def is_error_response(response):
error = response.get('error')
if error is None:
return False
print("API Error: "
f"code={error['code']} "
f"domain={error['errors'][0]['domain']} "
f"reason={error['errors'][0]['reason']} "
f"message={error['errors'][0]['message']!r}")
return True
that you'll invoke after each statement of form results = comments_data.json(). In case of the first occurrence of that statement, you'll have:
results = comments_data.json()
if is_error_response(results):
return []
nextPageToken = results.get("nextPageToken")
For the second instance of that statement:
results = comments_data.json()
if is_error_response(results):
return comments_list
nextPageToken = results.get("nextPageToken")
Notice that the function is_error_response above prints out an error message on stdout in case its argument in an API error response; this is for the purpose of having the user of your script informed about the API call failure.
How we can define multiple examples for request object just like we define examples for response object as below.
#ApiResponses(value = {
#ApiResponse(code = 200, response = Response.class, message = "Success", examples = #io.swagger.annotations.Example(
value = {
#ExampleProperty(value = "{'key1': 'value1', 'key2':'value2'}", mediaType = "application/json")
}))
})
I couldn't find much in swagger2, but i upgraded to openapi and then given examples as below
#io.swagger.v3.oas.annotations.parameters.RequestBody(content = {
#Content(mediaType = MediaType.APPLICATION_JSON_VALUE, examples = #ExampleObject(value = "{\"key1\": \"value1\", \"key2\":\"value2\", \"key3\":\"value3\"}"))
})
enhancing "Abhinav manthri" answer using "schema" and using "oneOf" for multiple requests
#io.swagger.v3.oas.annotations.parameters.RequestBody(content = {
#Content(mediaType = MediaType.APPLICATION_JSON_VALUE, schema = #Schema(oneOf = { MyReqBody1.class, MyReqBody2.class })) })
How would one set up logging each request to a different json file with RestRserve?
I tried using the lgr package (referred to in RestRserve's doc on logging) like so:
library(RestRserve)
library(lgr)
app = Application$new(content_type = "text/plain")
# RestRserve logger
app$logger = RestRserve::Logger$new(level = "trace", name = "mylogger",
printer=function(timestamp, level, logger_name, pid, message, ...)
{
lgr$log(level=tolower(level), msg=message, ...)
}
)
# JSON appender in lgr
tf <- tempfile(tmpdir="D:/temp", fileext=".log")
lgr$add_appender(AppenderJson$new(tf), name = "json")
# Endpoint
app$add_get("/sqrt", function(request, response) {
on.exit({
# Next log file
tf <- tempfile(tmpdir="D:/temp", fileext=".log")
lgr$appenders$json$set_file(tf)
})
app$logger$info(msg="", context=list(request_id = request$id, message="Process start"))
response$body = sqrt(x)
app$logger$info(msg="", context=list(request_id = request$id, message="Process end"))
})
# Submit request
request = Request$new(path = "/sqrt", method = "GET", parameters_query = list(x = "10"))
response = app$process_request(request)
But this splits up a request's log info across two files. I'm also quite sure it wouldn't work for simultaneous requests.
I believe you even don't need any special logger - just use writeLines. Also you can rely on req$id to name files since it is unique.
library(RestRserve)
req = Request$new()
res = Response$new()
fl = file.path(tempdir(), paste0(req$id, ".log"))
con = file(fl, open = "at")
writeLines("Process start", con)
res$set_body(sqrt(10))
writeLines("Process end", con)
close(con)
readLines(fl)
unlink(fl)
I'm trying to request some data via the mt gox API (mtgox.com) and theres some example code in python that I'd like to basically copy into R.
import hmac, base64, hashlib, urllib2
base = 'https://data.mtgox.com/api/2/'
def makereq(key, secret, path, data):
hash_data = path + chr(0) + data
secret = base64.b64decode(secret)
sha512 = hashlib.sha512
hmac = str(hmac.new(secret, hash_data, sha512))
header = {
'User-Agent': 'My-First-Trade-Bot',
'Rest-Key': key,
'Rest-Sign': base64.b64encode(hmac),
'Accept-encoding': 'GZIP',
}
return urllib2.Request(base + path, data, header)
I have some R code already
install.packages("base64")
install.packages("caTools")
install.packages("digest")
install.packages("RCurl")
library(RCurl)
library(caTools)
library(base64)
base<- "https://data.mtgox.com/api/2"
path<- "BTCUSD/money/ticker"
APIkey<-"******" #this is private but its a long hex number
secretAPIkey<-"*****" #this too, but this is in base64
makeReq<-function(key, secret, path, post_data)
{
browser()
message <- paste(path, NULL, post_data)
secret<-base64decode(secret,"character")
theHmac <-hmac(secret,message,"sha512")
header <-
{
c(
User.Agent = "My Bot",
Rest.Key = key,
Rest.Sign = base64encode(theHmac),
Acccept.encoding = "GZIP"
)
}
return (getURL(paste(base,path), post_data, header) )
}
I don't know how to get the "header" thing to work though, and I might be using getURL() incorrectly.
If you want to see the whole problem, the instructions are here https://bitbucket.org/nitrous/mtgox-api/overview, scroll down to the first block of code.
but I'm probably just making some elementary mistake with R header syntax...
try to use postForm (from RCurl) instead of getURL:
postForm(paste(base,path),
.opts = list(postfields = post_data,
useragent = 'R',
httpheader = c('Rest-Key' = key,
'Rest-Sign' = base64encode(theHmac)),
timeout = 4,
ssl.verifypeer = FALSE)
)