I am writing a Go server which handles image uploading via Multipart Requests, but after a while i get really bad memory usages (up to 100% RAM).
I am using the basic http.ListenAndServeTLS and http.HandleFunc to do so.
Below a snippet of my code and the svg i got from pprof after a lot of image uploads.
func HandleUploadImage(res http.ResponseWriter, req *http.Request) {
defer func() {
// Release mem
if req != nil {
fmt.Println("Releasing request")
if req.MultipartForm != nil {
fmt.Println("Releasing MultipartForm")
req.MultipartForm.RemoveAll()
}
if req.Body != nil {
fmt.Println("Closing req.Body")
req.Body.Close()
}
}
}()
// parse request
const _2MB = 1 << 20
if err := req.ParseMultipartForm(_2MB); nil != err {
u.Log("Error", "Couldn't parse MultiPartForm", map[string]interface{}{"Error": err}, u.GetFile_line(), "")
return
}
id := req.FormValue(formValueString)
// get the json data
if err := json.Unmarshal([]byte(id), marshalStruct); nil != err {
u.Log("Error", "Couldn't Unmarshal JSON", map[string]interface{}{"Error": err}, u.GetFile_line(), "")
return
}
// Do something with the image and save it
// Marshal the reponse to JSON.
resJSON, err := response.ToJSON()
if err != nil {
u.Log("Error", "Couldn't create JSON from response struct", map[string]interface{}{"Error": err, "ResponseStruct": response}, u.GetFile_line(), username)
return
}
}
pprof_image
Related
In the following codes, s.getFile gets file from S3 and return a struct of io.ReadCloser and ContentLength.
WriteResultesponse write the file to http.ResponseWriter.
But *reader.ContentLength sometimes is different from actualContentLength.
Any idea why? Thanks
s3Ctx, closeS3 := context.WithTimeout(ctx, xxx) // 1 hour
defer closeS3()
// directly stream result from locations
for _, location := range Locations {
reader, err := s.getFile(s3Ctx, xxx)
// reader is
//struct {
// Data io.ReadCloser
// ContentLength *int64
//}
if err != nil {
return err
}
actualContentLength, err := WriteResultesponse(params.Writer, ResultResponse{
Data: reader.Data,
})
}
// WriteResultResponse streams the result data to the user.
func WriteResultResponse(w http.ResponseWriter, resultResp ResultResponse) (int64, error) {
w.Header().Set("Content-Type", "text/plain")
// resultResp.Data is io.ReadCloser
defer resultResp.Data.Close()
return io.Copy(w, resultResp.Data)
}
UPDATE
How about
if f, ok := params.Writer.(http.Flusher); ok {
f.Flush()
}
?
I am trying to send a file and json data from a client to the server, but the server does not respond to the request and fails to decode the json data but the file is received
I am using a map and I send it as a json format(avsUpload), the reason for this is that the client can have a quantity of data n and a struc is not ideal
client code:
func UploadFile(file_up string,avsUpload map[string]string){
//get file
file, err := os.Open(file_up)
if err!=nil{
fmt.Println(red(" ERROR ")+"open file",file,err)
return
}
defer file.Close()
//get file info
fileInfo, err := os.Stat(file_up)
if err!=nil{
fmt.Println(red(" ERROR ")+"geting file info",err)
return
}
//create form
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", fileInfo.Name())
if err != nil {
fmt.Println(red(" ERROR ")+"creating form file",err)
return
}
io.Copy(part, file)
writer.Close()
//encode json, avsUpload as map
json.NewEncoder(body).Encode(avsUpload)
//create request
request, err := http.NewRequest("POST", "http://127.0.0.1:2047/ctrl/upload", body)
if err != nil {
fmt.Println(red(" ERROR "),err)
return
}
//add headers
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Content-Type","application/json; charset=utf-8")
request.Header.Add("Authorization", "BEARER "+readKey())
request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength,10))
//create req
client := &http.Client{}
response, err := client.Do(request)
if err != nil {
fmt.Println(red(" ERROR "),err)
return
}
defer response.Body.Close()
content, err := ioutil.ReadAll(response.Body)
if err != nil {
fmt.Println(red(" ERROR "),err)
}
fmt.Println(" "+string(content))
}
capturing the requests, I could see that json data is sent normally
server code:
func Upload(w http.ResponseWriter, r *http.Request){
//set header
w.Header().Set("Content-Type", "multipart/form-data")
w.Header().Set("Content-Type", "application/json")
//set max request size
r.Body = http.MaxBytesReader(w, r.Body, MaxFileSize)
fmt.Println("size>>> ",r.ContentLength)
//close conection if request is > MaxFileSize
if r.ContentLength > MaxFileSize {
http.Error(w, "File size is too large, max "+strconv.Itoa(FileSize)+" mb's\n", http.StatusExpectationFailed)
log.Error(w, "File size is too large, max "+strconv.Itoa(FileSize)+" mb's", http.StatusExpectationFailed)
return
}
//create miltipart reader
reader, err := r.MultipartReader()
if err != nil {
log.Error(w, err.Error(), http.StatusBadRequest)
return
}
// parse file form
p, err := reader.NextPart()
if err != nil && err != io.EOF {
log.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//check if te variable file exist in form
if p.FormName() != "file" {
http.Error(w, "file is expected\n", http.StatusBadRequest)
log.Error(w, "file is expected", http.StatusBadRequest)
return
}
//check file name length
if len(p.FileName()) > 100 {
http.Error(w, "file name is too long\n", http.StatusBadRequest)
log.Error(w, "file name is too long", http.StatusBadRequest)
return
}
//check if te filename contains spaces
var fileName string
if strings.Contains(p.FileName(), " "){
fileName=strings.Replace(p.FileName(), " ", "_", -1)
}else{
fileName=p.FileName()
}
//get user from id in token
_, claims, err := jwtauth.FromContext(r.Context())
if err != nil {
log.Error(w, err.Error(), http.StatusBadRequest)
return
}
user:=getUser(int(claims["id"].(float64)))
//create buffer
buf := bufio.NewReader(p)
//upload file to user dir
f, err := os.OpenFile("test/"+user+"/tmpfile/"+fileName, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
log.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer f.Close()
//decode json from client
avsSelect:=make(map[string]string)
err = json.NewDecoder(r.Body).Decode(&avsSelect)
if err != nil {
log.Error(w, err.Error(), http.StatusBadRequest)
return
}
//copy file to user dir
lmt := io.MultiReader(buf, io.LimitReader(p, MaxFileSize - 511))
fileSize, err := io.Copy(f, lmt)
if err != nil && err != io.EOF {
http.Error(w, "File size is too large, max "+strconv.Itoa(FileSize)+" mb's\n", http.StatusExpectationFailed)
log.Error(w, err.Error(), http.StatusInternalServerError)
os.Remove(f.Name())
return
}
defer p.Close()
//print conformation message
w.Write([]byte(fmt.Sprintf(green("SERVER: ")+"File "+fileName+" uploaded")))
fmt.Sprintf("File "+fileName+" uploaded")
log.Info("Size request: %#v\n", r.ContentLength)
log.Info("Size file uploaded: %#v\n",fileSize)
return
}
The server records the following for diferents request:
invalid character ' ' in literal false (expecting 'a')400
invalid character '¥' looking for beginning of value400"
invalid character '\\u0086' looking for beginning of value400"
It could be an issue with your JWT auth since you're ignoring that potential error coming from jwtauth.FromContext. Try adding error handling there and see if you get anything useful.
Your multipart construction seems wrong. You create the first part from a file, but you don't create a second part for the json, you decode it directly into to the buffer. Instead, you should use writer.CreatePart to create a new part, and write the json data to the writer returned from that.
I'm working on some tests in Go and I have spent the past 2 days trying to make it work but I couldn't. My problem is that the test returns 400 even when the user does exist.
This is my getUser function
func (handler *UserHandler) getUser(w http.ResponseWriter, ID int) {
logfile, err := os.OpenFile("events.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatalf("Error opening file: %v", err)
}
defer logfile.Close()
log.SetOutput(logfile)
user := db.Fetch(ID)
userJSON, err := json.Marshal(user)
if err != nil {
log.Printf("Error while marshaling the user into JSON: %v", err)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// userJSON is sent as http Response
w.Write(userJSON)
}
This is my UserHandler
type UserHandler struct{}
func (handle *UserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var head string
head, r.URL.Path = ShiftPath(r.URL.Path)
id, err := strconv.Atoi(head)
if err != nil {
http.Error(w, fmt.Sprintf("Invalid user ID %q", head), http.StatusBadRequest)
return
}
switch r.Method {
case "GET":
handle.getUser(w, id)
default:
http.Error(w, "Only GET is allowed", http.StatusMethodNotAllowed)
}
}
func ShiftPath(p string) (head, tail string) {
p = path.Clean("/" + p)
i := strings.Index(p[1:], "/") + 1
if i <= 0 {
return p[1:], "/"
}
return p[1:i], p[i:]
}
And this is my test
func TestGetUser(t *testing.T) {
handler := new(UserHandler)
mux := http.NewServeMux()
mux.HandleFunc("/user/", handler.ServeHTTP)
writer := httptest.NewRecorder()
request, _ := http.NewRequest("GET", "/user/12", nil)
mux.ServeHTTP(writer, request)
if writer.Code != 200 {
t.Errorf("Response code is %v", writer.Code)
}
}
Issue with code ====> id, err := strconv.Atoi(head)
Due to error you see a return and hence you see 400 error.
Have your server code fully functional with valid logic.
Suggestion: Always print or debug line by line. You can find the issue and root cause.
Hey there I would like to parse a http.resquest two times like below. When I parsed the Body the first time, the body will be closed. I need some help/hint what the best way is to handle this, do I have to create a copy of the request or is there a better way?
func myfunc(w http.ResponseWriter, req *http.Request) {
err := parseBody(req, &type1){
.....
}
err := parseBody(req, &type2){
.....
}
}
Thanks for help
It's true that you can read body only once and it's ok because to parse body more than once you don't have to read it more that one time. Let's consider simple example:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
)
type RequestData1 struct {
Code string `json:"code"`
Status string `json:"status"`
}
type RequestData2 struct {
Status string `json:"status"`
Message string `json:"message"`
}
func main() {
http.HandleFunc("/post", post)
http.ListenAndServe(":8080", nil)
}
If we use this code:
func post(w http.ResponseWriter, r *http.Request) {
body1, err := ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
}
rd1 := RequestData1{}
err = json.Unmarshal(body1, &rd1)
if err != nil {
panic(err)
}
body2, err := ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
}
rd2 := RequestData2{}
err = json.Unmarshal(body2, &rd2)
if err != nil {
panic(err) // panic!!!
}
fmt.Printf("rd1: %+v \nrd2: %+v", rd1, rd2)
w.WriteHeader(http.StatusOK)
w.Write([]byte(`Look into console.`))
}
we will have panic: http: panic serving [::1]:54581: unexpected end of JSON input
but with next code:
func post(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
}
rd1 := RequestData1{}
err = json.Unmarshal(body, &rd1)
if err != nil {
panic(err)
}
rd2 := RequestData2{}
err = json.Unmarshal(body, &rd2)
if err != nil {
panic(err)
}
fmt.Printf("rd1: %+v \nrd2: %+v", rd1, rd2)
w.WriteHeader(http.StatusOK)
w.Write([]byte(`Look into console.`))
}
all works! You can test it by issuing request:
curl -X POST 'http://localhost:8080/post' \
-H 'Content-Type: application/json' -d '{"code":"200", "status": "OK", "message": "200 OK"}'
Result will be:
rd1: {Code:200 Status:OK}
rd2: {Status:OK Message:200 OK}
When you read request.Body, you're reading the stream from the client (e.g. web browser). The client only sends the request once. If you want to parse it multiple times, read the whole thing out into a buffer (e.g. a []byte) and then parse that as many times as you want. Just be mindful of the potential memory use of many concurrent requests with large payloads, as you'll be holding the full payload in memory at least until you're fully done parsing it.
I would like to use golang post request, upload pictures, but I do not want to pass filepath, just want to pass [] byte
The following article are not what I need because they are used os.Open
golang POST data using the Content-Type multipart/form-data
func Upload(url, file string) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
// Add your image file
f, err := os.Open(file)
if err != nil {
return
}
defer f.Close()
fw, err := w.CreateFormFile("image", file)
if err != nil {
return
}
if _, err = io.Copy(fw, f); err != nil {
return
}
// Add the other fields
if fw, err = w.CreateFormField("key"); err != nil {
return
}
if _, err = fw.Write([]byte("KEY")); err != nil {
return
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
Since you use
if _, err = io.Copy(fw, f); err != nil {
return
}
You may as well edit your code to:
Add new import: "bytes"
Change the method signature to func Upload(url string, file []byte) (err error)
Use io.Copy(fw, bytes.NewReader(f))