Call golang jsonrpc with curl - http

I have "hello world" rpc service written in golang. It works fine and go jsonrpc client is working. But I need to send request with curl and this example doesn't work:
curl \
-X POST \
-H "Content-Type: application/json" \
-d '{"id": 1, "method": "Test.Say", "params": [{"greet": "world"}]}' \
http://localhost:1999/_goRPC_
Go accept connection but produce absolutely no result:
curl: (52) Empty reply from server
Here my go code:
package main
import (
"log"
"os"
"time"
"net"
"net/rpc"
"net/rpc/jsonrpc"
)
// RPC Api structure
type Test struct {}
// Greet method arguments
type GreetArgs struct {
Name string
}
// Grret message accept object with single param Name
func (test *Test) Greet(args *GreetArgs, result *string) (error) {
*result = "Hello " + args.Name
return nil
}
// Start server with Test instance as a service
func startServer(ch chan<- bool, port string) {
test := new(Test)
server := rpc.NewServer()
server.Register(test)
listener, err := net.Listen("tcp", ":" + port)
if err != nil {
log.Fatal("listen error:", err)
}
defer listener.Close()
for {
conn, err := listener.Accept()
if err != nil {
log.Fatal(err)
}
go server.ServeCodec(jsonrpc.NewServerCodec(conn))
ch <- true
}
}
// Start client and call Test.Greet method
func startClient(port string) {
conn, err := net.Dial("tcp", ":" + port)
if err != nil {
panic(err)
}
defer conn.Close()
c := jsonrpc.NewClient(conn)
var reply string
var args = GreetArgs{"world"}
err = c.Call("Test.Greet", args, &reply)
if err != nil {
log.Fatal("arith error:", err)
}
log.Println("Result: ", reply)
}
func main() {
if len(os.Args) < 2 {
log.Fatal("port not specified")
}
port := os.Args[1]
ch := make(chan bool)
go startServer(ch, port)
time.Sleep(500 * time.Millisecond)
go startClient(port)
// Produce log message each time connection closes
for {
<-ch
log.Println("Closed")
}
}

The jsonrpc package doesn't support json-rpc over HTTP currently. So, you can't call jsonrpc with curl. If you really want to do that, you can make a HTTP handler that adapts the HTTP request/response to a ServerCodec. For example:
package main
import (
"io"
"log"
"net"
"net/http"
"net/rpc"
"net/rpc/jsonrpc"
"os"
)
type HttpConn struct {
in io.Reader
out io.Writer
}
func (c *HttpConn) Read(p []byte) (n int, err error) { return c.in.Read(p) }
func (c *HttpConn) Write(d []byte) (n int, err error) { return c.out.Write(d) }
func (c *HttpConn) Close() error { return nil }
// RPC Api structure
type Test struct{}
// Greet method arguments
type GreetArgs struct {
Name string
}
// Grret message accept object with single param Name
func (test *Test) Greet(args *GreetArgs, result *string) error {
*result = "Hello " + args.Name
return nil
}
// Start server with Test instance as a service
func startServer(port string) {
test := new(Test)
server := rpc.NewServer()
server.Register(test)
listener, err := net.Listen("tcp", ":"+port)
if err != nil {
log.Fatal("listen error:", err)
}
defer listener.Close()
http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/test" {
serverCodec := jsonrpc.NewServerCodec(&HttpConn{in: r.Body, out: w})
w.Header().Set("Content-type", "application/json")
w.WriteHeader(200)
err := server.ServeRequest(serverCodec)
if err != nil {
log.Printf("Error while serving JSON request: %v", err)
http.Error(w, "Error while serving JSON request, details have been logged.", 500)
return
}
}
}))
}
func main() {
if len(os.Args) < 2 {
log.Fatal("port not specified")
}
port := os.Args[1]
startServer(port)
}
Now you can call it with curl -X POST -H "Content-Type: application/json" -d '{"id": 1, "method": "Test.Greet", "params": [{"name":"world"}]}' http://localhost:port/test
Part of the code is from this post

#jfly has a nifty solution.
Another option, if you still wanted to test with something besides the go jsonrpc cient (probably the easiest option), or use #jfly's answer, is you can use telnet to send raw data:
computer:~ User$ telnet 127.0.0.1 8888
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
{"method":"Test.Greet","params":[{"Name":"world"}],"id":0}
{"id":0,"result":"Hello world","error":null}
{"method":"Test.Greet","params":[{"Name":"world"}],"id":0}
{"id":0,"result":"Hello world","error":null}
{"method":"Test.Greet","params":[{"Name":"world"}],"id":0}
{"id":0,"result":"Hello world","error":null}
The above is the output including payload I typed in and your server's responses.
tcpdump was my friend when I was figuring out the right payload to send.

Related

Sending data in Chunks using single HTTP Post connection

I receive the contents of a file from a data source in chunks. As and when I receive the chunk I want to send the chunk data to a service using http POST request. And by keeping alive the same http POST connection used for sending the first chunk I want to send the remaining chunks of data.
I came up with the following code snippet to implement something similar.
Server-Side
func handle(w http.ResponseWriter, req *http.Request) {
buf := make([]byte, 256)
var n int
for {
n, err := req.Body.Read(buf)
if n == 0 && err == io.EOF {
break
}
fmt.Printf(string(buf[:n]))
}
fmt.Printf(string(buf[:n]))
fmt.Printf("Transfer Complete")
}
Client-Side
type alphaReader struct {
reader io.Reader
}
func newAlphaReader(reader io.Reader) *alphaReader {
return &alphaReader{reader: reader}
}
func (a *alphaReader) Read(p []byte) (int, error) {
n, err := a.reader.Read(p)
return n, err
}
func (a *alphaReader) Reset(str string) {
a.reader = strings.NewReader(str)
}
func (a *alphaReader) Close() error {
return nil
}
func main() {
tr := http.DefaultTransport
alphareader := newAlphaReader(strings.NewReader("First Chunk"))
client := &http.Client{
Transport: tr,
Timeout: 0,
}
req := &http.Request{
Method: "POST",
URL: &url.URL{
Scheme: "http",
Host: "localhost:8080",
Path: "/upload",
},
ProtoMajor: 1,
ProtoMinor: 1,
ContentLength: -1,
Body: alphareader,
}
fmt.Printf("Doing request\n")
_, err := client.Do(req)
alphareader.Reset("Second Chunk")
fmt.Printf("Done request. Err: %v\n", err)
}
Here I want that when I do alphareader.Reset("Second Chunk"), the string "Second Chunk" should be sent using the POST connection made earlier. But that is not happening. The connection gets closed after sending the First Chunk of data. Also I have not written the Close() method properly which I'm not sure how to implement.
I'm newbie to golang and any suggestions would be greatly helpful regarding the same.
A *strings.Reader returns io.EOF after the initial string has been read and your wrapper does nothing to change that, so it cannot be reused. You're looking for io.Pipe to turn the request body into an io.Writer.
package main
import (
"io"
"net/http"
)
func main() {
pr, pw := io.Pipe()
req, err := http.NewRequest("POST", "http://localhost:8080/upload", pr)
if err != nil {
// TODO: handle error
}
go func() {
defer pw.Close()
if _, err := io.WriteString(pw, "first chunk"); err != nil {
_ = err // TODO: handle error
}
if _, err := io.WriteString(pw, "second chunk"); err != nil {
_ = err // TODO: handle error
}
}()
res, err := http.DefaultClient.Do(req)
if err != nil {
// TODO: handle error
}
res.Body.Close()
}
Also, don't initialize the request using a struct literal. Use one of the constructors instead. In your code you're not setting the Host and Header fields, for instance.

How to get parameters in POST request

I am trying to get the parameters made in a POST request, but I am not able to make it, my code is:
package main
import (
"fmt"
"log"
"net/http"
)
func main() {
http.HandleFunc("/", hello)
fmt.Printf("Starting server for testing HTTP POST...\n")
if err := http.ListenAndServe(":8080", nil); err != nil {
log.Fatal(err)
}
}
func hello(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.Error(w, "404 not found.", http.StatusNotFound)
return
}
switch r.Method {
case "POST":
// Call ParseForm() to parse the raw query and update r.PostForm and r.Form.
if err := r.ParseForm(); err != nil {
fmt.Fprintf(w, "ParseForm() err: %v", err)
return
}
name := r.Form.Get("name")
age := r.Form.Get("age")
fmt.Print("This have been received:")
fmt.Print("name: ", name)
fmt.Print("age: ", age)
default:
fmt.Fprintf(w, "Sorry, only POST methods are supported.")
}
}
I am making the POST request in the terminal as follows:
curl -X POST -d '{"name":"Alex","age":"50"}' localhost:8080
And then the output is:
This have been received:name: age:
Why it is not taking the parameters? What I am doing wrong?
As you pass your body as a json object, you better define a Go struct matching that object and decode the request body to the object.
type Info struct {
Name string
Age int
}
info := &Info{}
if err := json.NewDecoder(r.Body).Decode(info); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_ = json.NewEncoder(w).Encode(info)
You can find the whole working code here.
$ curl -X POST -d '{"name":"Alex","age":50}' localhost:8080
This POST request is working fine now.
You could modify the Go struct and also the response object as you like .

How to send JSON inside JSON in POST

I am trying to send the following data via a POST HTTP request to an API:
{
"client_interface":{
"source_address":source,
"destination_address":destn,
"message":encrypted_msg,
"business_event_url":settings.Message_CallbackURL
},
"server_interface":{
"message_id":msg_id
}
}
The API is responding with the following error:
{
"Meta":{
"Requestid":12301343169471000
},
"Error":{
"Message":"Request body contains badly-formed JSON (at position 51)",
"Param":""
}
}
CODE:
apiUrl := "http://example.com"
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify : true},
}
jsonStr := []byte(`{
"client_interface": {
"source_address": source,
"destination_address": destn,
"message": encrypted_msg,
"business_event_url": settings.Message_CallbackURL
},
"server_interface": {
"message_id": msg_id
}
}`)
req, err := http.NewRequest("POST", apiUrl, bytes.NewBuffer(jsonStr))
fmt.Println("req..........",req)
if err!=nil{
log.Println("err in http req..............",err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("requestid", msg_id)
req.Header.Set("Authorization", "Bearer "+conn_token)
client := &http.Client{Transport: tr}
resp, err := client.Do(req)
if resp!=nil{
body, _ := ioutil.ReadAll(resp.Body)
}
Using struct :
package main
import (
"fmt"
"net/http"
"io/ioutil"
"bytes"
//"crypto/tls"
"encoding/json"
)
type client_interface struct {
source_address string `json:"string"`
destination_address uint64 `json:"uint64"`
message string `json:"string"`
business_event_url string `json:"string"`
}
type server_interface struct {
message_id uint64 `json:"uint64"`
}
type data struct {
client_interface client_interface `json:"client_interface"`
server_interface server_interface `json:"server_interface"`
}
func main() {
url := "https://example.com"
fmt.Println("URL:>", url)
client_interface := client_interface{}
server_interface := server_interface{}
client_interface.source_address="1"
client_interface.destination_address=1111111111
client_interface.message="khsjhdjks"
client_interface.business_event_url="http://callbackurl-hdfc"
server_interface.message_id=8210993557215399651
fmt.Println("server_interface..........",server_interface)
fmt.Println("client_interface..........",client_interface)
body1 := &data{
client_interface: client_interface,
server_interface: server_interface,
}
fmt.Println("body1..........",body1)
t,e:=json.Marshal(body1)
fmt.Println("t..........",t)
fmt.Println("e..........",e)
req, err := http.NewRequest("POST", url, bytes.NewReader(t))
fmt.Println("req......",req)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("requestid", "8210993557215399651")
req.Header.Set("Authorization", "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6IkJCOENlRlZxeWFHckdOdWVoSklpTDRkZmp6dyIsImtpZCI6IkJCOENlRlZxeWFHckdOdWVoSklpTDRkZmp6dyJ9.eyJhdWQiOiJhcGk6Ly90cC1kZXYtdGFubGEtYXBpIiwiaXNzIjoiaHR0cHM6Ly9zdHMud2luZG93cy5uZXQvY2JhYThhYmItZTcwZi00YmI4LWIwNDQtZmZiZjAwNzk0NzkwLyIsImlhdCI6MTU3NTg5MTI3NCwibmJmIjoxNTc1ODkxMjc0LCJleHAiOjE1NzU4OTUxNzQsImFjciI6IjEiLCJhaW8iOiI0MlZnWU9EY3JjenlhZXIxdkRMRDVlNHVtWUxha1UrRUplOVYrZGVlRFgrOTNUMytNRGNBIiwiYW1yIjpbInB3ZCJdLCJhcHBpZCI6IjFmMjI1N2ZlLWIzYjktNGQ2Ny05M2YyLWRjNjM2N2Q2MGM4MCIsImFwcGlkYWNyIjoiMCIsImlwYWRkciI6IjE0LjE0My4xODcuMjUwIiwibmFtZSI6ImhkZmMuMTEiLCJvaWQiOiIzOGQxMGFlNS01OGYyLTQ0NjUtYTFkOC04YTc0NDAzYjc5MmEiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiIzNDdUd0ZwYUw5MDhmOXlNRWlGOWNHMU84THFQYmJxZk45VzhyQWVEX1prIiwidGlkIjoiY2JhYThhYmItZTcwZi00YmI4LWIwNDQtZmZiZjAwNzk0NzkwIiwidW5pcXVlX25hbWUiOiJoZGZjLjExQFRhbmxhUHJvZHVjdC5vbm1pY3Jvc29mdC5jb20iLCJ1cG4iOiJoZGZjLjExQFRhbmxhUHJvZHVjdC5vbm1pY3Jvc29mdC5jb20iLCJ1dGkiOiJuS05TTXRsT3VFeXMtQjRIOGJ3TEFRIiwidmVyIjoiMS4wIn0.F5H9WCOktau3JaqNyWM91A5jFpJ9eJE99fBWvqDq9kOfCk3OCJnHFKXtIaIA7MoqbxWpNZt1yWpVKuw8gd2Lg_9nfUvvXts2DJHVQN0EqQmFUyWTzhdLW8ZVi6E9RtXK2aEWrI2TVceL5C2wbYOQYfvV4LzjTuNbs6k_20cQ0nD6oO1Id16VVFQWy9yKvpDzsTrvlQdFBZeohIfyL9XWKa8DOk0gxe4bjC7OFmuMsF3FZE5XPaQPHOJ3ejlZJiApml2TlRHnvLpkn1biE3NTAu9aO2lE262lyLg8ZaU0sbPuQaS8P797a-outxLvKEMh07895mA9g6vMxEdRV9X2eA")
client := &http.Client{}
resp, err := client.Do(req)
fmt.Println("err.............",err)
if err != nil {
panic(err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println("response Body:", string(body))
}
First of all: please use gofmt
Your first code can't work because golang doesn't substitute variables inside string. It's better to use structs.
With structs it's not working because you named struct fields from lower case latter, it means "private" fields in go. So encoding/json package can't access them and just skip them. Use Capital first letters.
Another fix is about 'json:"tag"' - here tag means encoded field name, not type. So instead of 'json:"string"' you should use 'json:"message_id"' or so. You can specify type like this 'json:"field_name,type"' or like this 'json:",type"' but encoding/json guess type on his own.
(I used wrong quotes in tags because of markdown)
I used netcat -l 5000 to listen on 5000 port on localhost and print everything to the terminal. Then I changed url to http://localhost:5000 (not https) to send request to myself.
You need to restart netcat each time to work.
And I made logging a bit more readable.
Also it's CamelCase naming convention in go.
Changed your code a little bit
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"net/http"
//"crypto/tls"
"encoding/json"
"github.com/nikandfor/tlog"
)
type ClientInterface struct {
SourceAddress string `json:"source_address"`
DestinationAddress uint64 `json:"destination_address"`
Message string `json:"message"`
BusinessEventURL string `json:"business_event_url"`
}
type ServerInterface struct {
MessageID uint64 `json:"message_id"`
}
type Data struct {
ClientInterface ClientInterface `json:"client_interface"`
ServerInterface ServerInterface `json:"server_interface"`
}
var (
// use command line flag. so run like so:
// go run ./file.go -addr https://example.com
addr = flag.String("addr", "http://localhost:5000", "address to send data to")
)
func main() {
flag.Parse() // DO NOT FORGET TO PARSE FLAGS
fmt.Println("URL:>", *addr)
clientInterface := ClientInterface{
SourceAddress: "1",
DestinationAddress: 8886121111,
Message: "khsjhdjks",
BusinessEventURL: "http://callbackurl-hdfc",
}
serverInterface := ServerInterface{
MessageID: 8210993557215399651,
}
tlog.Printf("server_interface %+v", serverInterface)
tlog.Printf("client_interface %+v", clientInterface)
body1 := &Data{
ClientInterface: clientInterface,
ServerInterface: serverInterface,
}
tlog.Printf("body %+v", body1)
t, err := json.Marshal(body1)
if err != nil {
panic(err)
}
tlog.Printf("marshalled: %s", t)
req, err := http.NewRequest("POST", *addr, bytes.NewReader(t))
tlog.Printf("req %v", req)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("requestid", "8210993557215399651")
req.Header.Set("Authorization", "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6IkJCOENlRlZxeWFHckdOdWVoSklpTDRkZmp6dyIsImtpZCI6IkJCOENlRlZxeWFHckdOdWVoSklpTDRkZmp6dyJ9.eyJhdWQiOiJhcGk6Ly90cC1kZXYtdGFubGEtYXBpIiwiaXNzIjoiaHR0cHM6Ly9zdHMud2luZG93cy5uZXQvY2JhYThhYmItZTcwZi00YmI4LWIwNDQtZmZiZjAwNzk0NzkwLyIsImlhdCI6MTU3NTg5MTI3NCwibmJmIjoxNTc1ODkxMjc0LCJleHAiOjE1NzU4OTUxNzQsImFjciI6IjEiLCJhaW8iOiI0MlZnWU9EY3JjenlhZXIxdkRMRDVlNHVtWUxha1UrRUplOVYrZGVlRFgrOTNUMytNRGNBIiwiYW1yIjpbInB3ZCJdLCJhcHBpZCI6IjFmMjI1N2ZlLWIzYjktNGQ2Ny05M2YyLWRjNjM2N2Q2MGM4MCIsImFwcGlkYWNyIjoiMCIsImlwYWRkciI6IjE0LjE0My4xODcuMjUwIiwibmFtZSI6ImhkZmMuMTEiLCJvaWQiOiIzOGQxMGFlNS01OGYyLTQ0NjUtYTFkOC04YTc0NDAzYjc5MmEiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiIzNDdUd0ZwYUw5MDhmOXlNRWlGOWNHMU84THFQYmJxZk45VzhyQWVEX1prIiwidGlkIjoiY2JhYThhYmItZTcwZi00YmI4LWIwNDQtZmZiZjAwNzk0NzkwIiwidW5pcXVlX25hbWUiOiJoZGZjLjExQFRhbmxhUHJvZHVjdC5vbm1pY3Jvc29mdC5jb20iLCJ1cG4iOiJoZGZjLjExQFRhbmxhUHJvZHVjdC5vbm1pY3Jvc29mdC5jb20iLCJ1dGkiOiJuS05TTXRsT3VFeXMtQjRIOGJ3TEFRIiwidmVyIjoiMS4wIn0.F5H9WCOktau3JaqNyWM91A5jFpJ9eJE99fBWvqDq9kOfCk3OCJnHFKXtIaIA7MoqbxWpNZt1yWpVKuw8gd2Lg_9nfUvvXts2DJHVQN0EqQmFUyWTzhdLW8ZVi6E9RtXK2aEWrI2TVceL5C2wbYOQYfvV4LzjTuNbs6k_20cQ0nD6oO1Id16VVFQWy9yKvpDzsTrvlQdFBZeohIfyL9XWKa8DOk0gxe4bjC7OFmuMsF3FZE5XPaQPHOJ3ejlZJiApml2TlRHnvLpkn1biE3NTAu9aO2lE262lyLg8ZaU0sbPuQaS8P797a-outxLvKEMh07895mA9g6vMxEdRV9X2eA")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
tlog.Printf("response Status: %v", resp.Status)
tlog.Printf("response Headers: %v", resp.Header)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
tlog.Printf("response Body: %s", string(body))
}
$ nc -l 5000
POST / HTTP/1.1
Host: localhost:5000
User-Agent: Go-http-client/1.1
Content-Length: 92
Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6IkJCOENlRlZxeWFHckdOdWVoSklpTDRkZmp6dyIsImtpZCI6IkJCOENlRlZxeWFHckdOdWVoSklpTDRkZmp6dyJ9.eyJhdWQiOiJhcGk6Ly90cC1kZXYtdGFubGEtYXBpIiwiaXNzIjoiaHR0cHM6Ly9zdHMud2luZG93cy5uZXQvY2JhYThhYmItZTcwZi00YmI4LWIwNDQtZmZiZjAwNzk0NzkwLyIsImlhdCI6MTU3NTg5MTI3NCwibmJmIjoxNTc1ODkxMjc0LCJleHAiOjE1NzU4OTUxNzQsImFjciI6IjEiLCJhaW8iOiI0MlZnWU9EY3JjenlhZXIxdkRMRDVlNHVtWUxha1UrRUplOVYrZGVlRFgrOTNUMytNRGNBIiwiYW1yIjpbInB3ZCJdLCJhcHBpZCI6IjFmMjI1N2ZlLWIzYjktNGQ2Ny05M2YyLWRjNjM2N2Q2MGM4MCIsImFwcGlkYWNyIjoiMCIsImlwYWRkciI6IjE0LjE0My4xODcuMjUwIiwibmFtZSI6ImhkZmMuMTEiLCJvaWQiOiIzOGQxMGFlNS01OGYyLTQ0NjUtYTFkOC04YTc0NDAzYjc5MmEiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiIzNDdUd0ZwYUw5MDhmOXlNRWlGOWNHMU84THFQYmJxZk45VzhyQWVEX1prIiwidGlkIjoiY2JhYThhYmItZTcwZi00YmI4LWIwNDQtZmZiZjAwNzk0NzkwIiwidW5pcXVlX25hbWUiOiJoZGZjLjExQFRhbmxhUHJvZHVjdC5vbm1pY3Jvc29mdC5jb20iLCJ1cG4iOiJoZGZjLjExQFRhbmxhUHJvZHVjdC5vbm1pY3Jvc29mdC5jb20iLCJ1dGkiOiJuS05TTXRsT3VFeXMtQjRIOGJ3TEFRIiwidmVyIjoiMS4wIn0.F5H9WCOktau3JaqNyWM91A5jFpJ9eJE99fBWvqDq9kOfCk3OCJnHFKXtIaIA7MoqbxWpNZt1yWpVKuw8gd2Lg_9nfUvvXts2DJHVQN0EqQmFUyWTzhdLW8ZVi6E9RtXK2aEWrI2TVceL5C2wbYOQYfvV4LzjTuNbs6k_20cQ0nD6oO1Id16VVFQWy9yKvpDzsTrvlQdFBZeohIfyL9XWKa8DOk0gxe4bjC7OFmuMsF3FZE5XPaQPHOJ3ejlZJiApml2TlRHnvLpkn1biE3NTAu9aO2lE262lyLg8ZaU0sbPuQaS8P797a-outxLvKEMh07895mA9g6vMxEdRV9X2eA
Content-Type: application/json
Requestid: 8210993557215399651
Accept-Encoding: gzip
{"client_interface":{"source_address":"1","destination_address":8886121111,"message":"khsjhdjks","business_event_url":"http://callbackurl-hdfc"},"server_interface":{"message_id":8210993557215399651}}
Is it what you've expected?
And the last. I strongly suggest you to read https://golang.org/doc/effective_go.html

Sending HTTP response is blocked until shell command finishes

Idea: there is a web server ready to receive messages that will trigger the execution of commands/tests on the server. I have started with a simple case where a simple ping is executed. The code below handles POST messages sent to /ping which contain the following json format:
{ "ip": "valid_ip_addr", "count": "4" }
The server will then run the command ping -c 4 valid_ip_address
Desired outcome: if the command can .Start() send back a 200 OK. If there are problems, send back an error message.
Problem: I send a 200 OK response right after checking that .Start() didn't give any errors, but this is being received after the command has finished.
Code: There are three functions: main(), handler() and ping(). The problem takes place in the last one.
package main
import (
"bufio"
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
)
var err error
type Ping struct {
Count string `json:"count"`
Ip string `json:"ip"`
}
func main() {
http.HandleFunc("/ping", handler)
http.ListenAndServe(":5050", nil)
}
func handler(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "POST":
p := Ping{}
err := json.NewDecoder(r.Body).Decode(&p)
if err != nil {
fmt.Printf("400 Bad request. Problem decoding the received json.\nDetails:\n%s\n", err.Error())
http.Error(w, err.Error(), 400)
return
}
fmt.Println("POST /ping ", p)
ping(w, p)
default:
http.Error(w, "Only POST is accepted.", 501)
}
}
func ping(w http.ResponseWriter, a Ping) {
cmdName := "ping"
cmdArgs := []string{"-c", a.Count, a.Ip}
cmd := exec.Command(cmdName, cmdArgs...)
cmdReader, err := cmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err)
http.Error(w, "Error creating StdoutPipe for Cmd\n"+err.Error(), 500)
return
}
// the following is used to print output of the command
// as it makes progress...
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
fmt.Printf("%s\n", scanner.Text())
//
// TODO:
// send output to server
}
}()
err = cmd.Start()
if err != nil {
fmt.Fprintln(os.Stderr, "Error starting Cmd", err)
http.Error(w, "Error starting Cmd\n"+err.Error(), 500)
return
}
// send 200 OK
fmt.Fprintf(w, "ping started")
err = cmd.Wait()
if err != nil {
fmt.Fprintln(os.Stderr, "Error waiting for Cmd", err)
}
}
curl for testing
curl -X POST http://localhost:5050/ping -d '{"ip": "127.0.0.1", "count": "4"}'
I would recommend to use select with timeout waiting for an error. Check out the following code.
func ping(w http.ResponseWriter, a Ping) {
cmdName := "ping"
cmdArgs := []string{"-c", a.Count, a.Ip}
cmd := exec.Command(cmdName, cmdArgs...)
cmdReader, err := cmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err)
http.Error(w, "Error creating StdoutPipe for Cmd\n"+err.Error(), 500)
return
}
// the following is used to print output of the command
// as it makes progress...
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
fmt.Printf("%s\n", scanner.Text())
//
// TODO:
// send output to server
}
}()
err = cmd.Start()
if err != nil {
fmt.Fprintln(os.Stderr, "Error starting Cmd", err)
http.Error(w, "Error starting Cmd\n"+err.Error(), 500)
return
}
// not sending response here anymore. Using the channel instead
errChan := make(chan error)
go func(ec chan error) {
err = cmd.Wait()
if err != nil {
errChan <- err
}
}(errChan)
select {
case err := <-errChan:
http.Error(w, "Error: "+err.Error(), 500)
// timeout 50ms just in case. But I presume you would get an error (if there is one in cmd) even before execution will get to this point
case <-time.After(time.Millisecond * 50):
fmt.Fprintf(w, "ping started")
}
}

Getting "127.0.0.1 can't assign requested address" - http.Client

What I'm doing is fairly straight-forward. I need to create a "proxy" server that is very minimal and fast. Currently I have a baseline server that is proxied to (nodejs) and a proxy-service (go). Please excuse the lack of actual "proxy'ing" - just testing for now.
Baseline Service
var http = require('http');
http.createServer(function (req, res) {
// console.log("received request");
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('Hello World\n');
}).listen(8080, '127.0.0.1');
console.log('Server running at http://127.0.0.1:8080/');
Proxy Service
package main
import (
"flag"
"log"
"net/http"
"net/url"
)
var (
listen = flag.String("listen", "0.0.0.0:9000", "listen on address")
logp = flag.Bool("log", false, "enable logging")
)
func main() {
flag.Parse()
proxyHandler := http.HandlerFunc(proxyHandlerFunc)
log.Fatal(http.ListenAndServe(*listen, proxyHandler))
log.Println("Started router-server on 0.0.0.0:9000")
}
func proxyHandlerFunc(w http.ResponseWriter, r *http.Request) {
// Log if requested
if *logp {
log.Println(r.URL)
}
/*
* Tweak the request as appropriate:
* - RequestURI may not be sent to client
* - Set new URL
*/
r.RequestURI = ""
u, err := url.Parse("http://localhost:8080/")
if err != nil {
log.Fatal(err)
}
r.URL = u
// And proxy
// resp, err := client.Do(r)
c := make(chan *http.Response)
go doRequest(c)
resp := <-c
if resp != nil {
err := resp.Write(w)
if err != nil {
log.Println("Error writing response")
} else {
resp.Body.Close()
}
}
}
func doRequest(c chan *http.Response) {
// new client for every request.
client := &http.Client{}
resp, err := client.Get("http://127.0.0.1:8080/test")
if err != nil {
log.Println(err)
c <- nil
} else {
c <- resp
}
}
My issue, as mentioned within the title, is that I am getting errors stating 2013/10/28 21:22:30 Get http://127.0.0.1:8080/test: dial tcp 127.0.0.1:8080: can't assign requested address from the doRequest function, and I have no clue why. Googling this particular error yields seemingly irrelevant results.
There are 2 major problems with this code.
You are not handling the client stalling or using keep alives (handled below by getTimeoutServer)
You are not handling the server (what your http.Client is talking to) timing out (handled below by TimeoutConn).
This is probably why you are exhausting your local ports. I know from past experience node.js will keep-alive you very aggressively.
There are lots of little issues, creating objects every-time when you don't need to. Creating unneeded goroutines (each incoming request is in its own goroutine before you handle it).
Here is a quick stab (that I don't have time to test well). Hopefully it will put you on the right track: (You will want to upgrade this to not buffer the responses locally)
package main
import (
"bytes"
"errors"
"flag"
"fmt"
"log"
"net"
"net/http"
"net/url"
"runtime"
"strconv"
"time"
)
const DEFAULT_IDLE_TIMEOUT = 5 * time.Second
var (
listen string
logOn bool
localhost, _ = url.Parse("http://localhost:8080/")
client = &http.Client{
Transport: &http.Transport{
Proxy: NoProxyAllowed,
Dial: func(network, addr string) (net.Conn, error) {
return NewTimeoutConnDial(network, addr, DEFAULT_IDLE_TIMEOUT)
},
},
}
)
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.StringVar(&listen, "listen", "0.0.0.0:9000", "listen on address")
flag.BoolVar(&logOn, "log", true, "enable logging")
flag.Parse()
server := getTimeoutServer(listen, http.HandlerFunc(proxyHandlerFunc))
log.Printf("Starting router-server on %s\n", listen)
log.Fatal(server.ListenAndServe())
}
func proxyHandlerFunc(w http.ResponseWriter, req *http.Request) {
if logOn {
log.Printf("%+v\n", req)
}
// Setup request URL
origURL := req.URL
req.URL = new(url.URL)
*req.URL = *localhost
req.URL.Path, req.URL.RawQuery, req.URL.Fragment = origURL.Path, origURL.RawQuery, origURL.Fragment
req.RequestURI, req.Host = "", req.URL.Host
// Perform request
resp, err := client.Do(req)
if err != nil {
w.WriteHeader(http.StatusBadGateway)
w.Write([]byte(fmt.Sprintf("%d - StatusBadGateway: %s", http.StatusBadGateway, err)))
return
}
defer resp.Body.Close()
var respBuffer *bytes.Buffer
if resp.ContentLength != -1 {
respBuffer = bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
} else {
respBuffer = new(bytes.Buffer)
}
if _, err = respBuffer.ReadFrom(resp.Body); err != nil {
w.WriteHeader(http.StatusBadGateway)
w.Write([]byte(fmt.Sprintf("%d - StatusBadGateway: %s", http.StatusBadGateway, err)))
return
}
// Write result of request
headers := w.Header()
var key string
var val []string
for key, val = range resp.Header {
headers[key] = val
}
headers.Set("Content-Length", strconv.Itoa(respBuffer.Len()))
w.WriteHeader(resp.StatusCode)
w.Write(respBuffer.Bytes())
}
func getTimeoutServer(addr string, handler http.Handler) *http.Server {
//keeps people who are slow or are sending keep-alives from eating all our sockets
const (
HTTP_READ_TO = DEFAULT_IDLE_TIMEOUT
HTTP_WRITE_TO = DEFAULT_IDLE_TIMEOUT
)
return &http.Server{
Addr: addr,
Handler: handler,
ReadTimeout: HTTP_READ_TO,
WriteTimeout: HTTP_WRITE_TO,
}
}
func NoProxyAllowed(request *http.Request) (*url.URL, error) {
return nil, nil
}
//TimeoutConn-------------------------
//Put me in my own TimeoutConn.go ?
type TimeoutConn struct {
net.Conn
readTimeout, writeTimeout time.Duration
}
var invalidOperationError = errors.New("TimeoutConn does not support or allow .SetDeadline operations")
func NewTimeoutConn(conn net.Conn, ioTimeout time.Duration) (*TimeoutConn, error) {
return NewTimeoutConnReadWriteTO(conn, ioTimeout, ioTimeout)
}
func NewTimeoutConnReadWriteTO(conn net.Conn, readTimeout, writeTimeout time.Duration) (*TimeoutConn, error) {
this := &TimeoutConn{
Conn: conn,
readTimeout: readTimeout,
writeTimeout: writeTimeout,
}
now := time.Now()
err := this.Conn.SetReadDeadline(now.Add(this.readTimeout))
if err != nil {
return nil, err
}
err = this.Conn.SetWriteDeadline(now.Add(this.writeTimeout))
if err != nil {
return nil, err
}
return this, nil
}
func NewTimeoutConnDial(network, addr string, ioTimeout time.Duration) (net.Conn, error) {
conn, err := net.DialTimeout(network, addr, ioTimeout)
if err != nil {
return nil, err
}
if conn, err = NewTimeoutConn(conn, ioTimeout); err != nil {
return nil, err
}
return conn, nil
}
func (this *TimeoutConn) Read(data []byte) (int, error) {
this.Conn.SetReadDeadline(time.Now().Add(this.readTimeout))
return this.Conn.Read(data)
}
func (this *TimeoutConn) Write(data []byte) (int, error) {
this.Conn.SetWriteDeadline(time.Now().Add(this.writeTimeout))
return this.Conn.Write(data)
}
func (this *TimeoutConn) SetDeadline(time time.Time) error {
return invalidOperationError
}
func (this *TimeoutConn) SetReadDeadline(time time.Time) error {
return invalidOperationError
}
func (this *TimeoutConn) SetWriteDeadline(time time.Time) error {
return invalidOperationError
}
We ran into this and after a lot of time trying to debug, I came across this: https://code.google.com/p/go/source/detail?r=d4e1ec84876c
This shifts the burden onto clients to read their whole response
bodies if they want the advantage of reusing TCP connections.
So be sure you read the entire body before closing, there are a couple of ways to do it. This function can come in handy to close to let you see whether you have this issue by logging the extra bytes that haven't been read and cleaning the stream out for you so it can reuse the connection:
func closeResponse(response *http.Response) error {
// ensure we read the entire body
bs, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
log.Println("Error during ReadAll!!", err2)
}
if len(bs) > 0 {
log.Println("Had to read some bytes, not good!", bs, string(bs))
}
return response.Body.Close()
}
Or if you really don't care about the body, you can just discard it with this:
io.Copy(ioutil.Discard, response.Body)
I have encountered this problem too, and i add an option {DisableKeepAlives: true} to http.Transport fixed this issue, you can have a try.
I came here when running a massive amount of SQL queries per second on a system without limiting the number of idle connections over a long period of time. As pointed out in this issue comment on github explicitly setting db.SetMaxIdleConns(5) completely solved my problem.

Resources