golang unix domain socket based http server does not work - http

I am trying to write a simple Unix domain socket based http server in golang. Here is the code:
package main
import (
"fmt"
"log"
"net/http"
"net"
"os"
"encoding/json"
"os/signal"
"syscall"
)
type UnixListener struct {
ServerMux *http.ServeMux
SocketPath string
UID int
GID int
SocketFileMode os.FileMode
}
//////////////////////////////////////////////////////////////////////
// Start the HTTP server with UNIX socket.
//////////////////////////////////////////////////////////////////////
func (l *UnixListener) Start() error {
listener, err := net.Listen("unix", l.SocketPath)
if err != nil {
return err
}
if err := os.Chown(l.SocketPath, l.UID, l.GID); err != nil {
return err
}
if err := os.Chmod(l.SocketPath, l.SocketFileMode); err != nil {
return err
}
go func(){
shutdown(listener)
}()
svr := http.Server{
Handler: l.ServerMux,
}
if err := svr.Serve(listener); err != nil {
return err
} else {
return nil
}
}
//////////////////////////////////////////////////////////////////////
// Shutdown the HTTP server.
//////////////////////////////////////////////////////////////////////
func shutdown(listener net.Listener) {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
s := <-c
listener.Close()
log.Fatalf("[FATAL] Caught a signal. sinal=%s", s)
}
func hello(w http.ResponseWriter, r *http.Request) {
log.Println("Received request %s", r.RequestURI)
fmt.Fprintf(w, "Hello World!\n")
w.Header().Set("ContentType", "application/json")
w.WriteHeader(http.StatusOK)
data := map[string]string { "name" : "satish"}
resp, _ := json.Marshal(data)
if _, err := w.Write(resp); err != nil {
fmt.Printf("Error writing response")
}
}
func main() {
mux := http.NewServeMux()
mux.HandleFunc("/hello", hello)
unixListener := UnixListener{
ServerMux: mux,
SocketPath: "/temp/test.sock",
UID: 501,
GID: 20,
SocketFileMode: 0644,
}
if err := unixListener.Start(); err != nil {
log.Fatalf("[FATAL] HTTP server error: %s", err)
}
}
But when I send the request to this UDS based server like below, the handler does not seem to be invoked, giving 404 not found error. What is the right way to implement UDS based http server and how to send the http request to it?
curl -vvv --unix-socket /temp/test.sock http:/hello
* Trying /Users/sburnwal/Projects/ACI/temp/test.sock:0...
* Connected to hello (/temp/test.sock) port 80 (#0)
> GET / HTTP/1.1
> Host: hello
> User-Agent: curl/7.84.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 404 Not Found
< Content-Type: text/plain; charset=utf-8
< X-Content-Type-Options: nosniff
< Date: Mon, 14 Nov 2022 02:38:11 GMT
< Content-Length: 19
<
404 page not found
* Connection #0 to host hello left intact

It appears it's interpreting http:/hello in your command line as the host name and using that for the header. And then requesting / as the path
Try replacing http:/hello in your command line with http://hello/hello. I'm not sure if the single slash vs double-slash is significant. It just looks suspicious.
curl -vvv --unix-socket /temp/test.sock http://hello/hello
Or change your Go code to have a response handler for "/" instead of just for "/hello"
mux.HandleFunc("/hello", hello)
mux.HandleFunc("/", hello)

Related

Golang program after making Get request returns HTTP Status code 503 or pending

I try to get csv data from kibana server by making get request.
After this server send me response with data which i write the csv file and save somewhere.
Every time csv file with single line "pending" inside.
Kibana logs shows Status code 503
But if i put url in browser i can get csv file with correct data.
Probably i need to wait more for response from kibana.
From my perspective the problem is next: server can send me the response with single line "pending" which means that it needs more time for preparing right response.
I tried to increase client time but it doesn't work as well
client := http.Client{
Timeout: 10 * time.Second,
}
The Idea with go routines and channels/wait group was next: they force Get request to wait for getting right data instead of "pending" and 503 status code
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"sync"
)
type KibanaReportResponse struct {
Path string `json:"path"`
}
var urlKibanaBase = "http://localhost:5601"
var urlKibanaPost = urlKibanaBase + "/api/reporting/generate/csv_searchsource?"
var urlParameters = "jobParams=%28browserTimezone%3AEurope%2FBerlin%2Ccolumns%3A%21%28%29%2CobjectType%3Asearch%2CsearchSource%3A%28fields%3A%21%28%28field%3A%27%2A%27%2Cinclude_unmapped%3Atrue%29%29%2Cindex%3Aec074c00-1f62-11ec-8056-8d208a1f6e77%2Cparent%3A%28filter%3A%21%28%29%2Cindex%3Aec074c00-1f62-11ec-8056-8d208a1f6e77%2Cquery%3A%28language%3Akuery%2Cquery%3A%27%27%29%29%2Csort%3A%21%28%28_score%3Adesc%29%29%2CtrackTotalHits%3A%21t%2Cversion%3A%21t%29%2Ctitle%3A%27Discover%20search%20%5B2021-09-27T09%3A19%3A44.977%2B02%3A00%5D%27%29"
var urlWithParam = urlKibanaPost + urlParameters
func main() {
var wg sync.WaitGroup
wg.Add(1)
pathCsvFile := getCsvPathFromKibana(urlWithParam)
go getCsvFile(urlKibanaBase, pathCsvFile, &wg)
defer wg.Wait()
}
func getCsvPathFromKibana(urlKib string) string {
resKibana := KibanaReportResponse{}
client := &http.Client{}
if req, err := http.NewRequest("POST", urlKib, nil); err != nil {
log.Println("Given a method, URL, andoptional body are wrong", err)
} else {
req.Header.Add("kbn-xsrf", "true")
if res, err := client.Do(req); err != nil {
log.Println("Probably problems depends on client policy or HTTP connection", err)
} else {
if err := json.NewDecoder(res.Body).Decode(&resKibana); err != nil {
log.Println("Problem by Json decoding \n", err)
} else {
return resKibana.Path
}
}
}
return resKibana.Path
}
func getCsvFile(urlKibanaBase string, pathCsvFile string, wg *sync.WaitGroup) error {
defer wg.Done()
res, err := http.Get(urlKibanaBase + pathCsvFile)
if err != nil {
return err
}
defer res.Body.Close()
switch res.StatusCode {
case 200:
dataBody, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
err = ioutil.WriteFile("data.csv", dataBody, 0666)
if err != nil {
return err
}
return nil
case 503:
fmt.Println("probably 503/pending")
return errors.New("probably 503/pending")
}
return nil
}
curl request
curl -v localhost:5601/api/reporting/jobs/download/ku5k3rxz00xs7fac46c0k12u
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 5601 (#0)
> GET /api/reporting/jobs/download/ku5k3rxz00xs7fac46c0k12u HTTP/1.1
> Host: localhost:5601
> User-Agent: curl/7.68.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< kbn-csv-contains-formulas: true
< kbn-max-size-reached: false
< content-disposition: inline; filename="Discover search [2021-09-27T09:19:44.977+02:00].csv"
< content-type: text/csv; charset=utf-8
< x-content-type-options: nosniff
< referrer-policy: no-referrer-when-downgrade
< kbn-name: 3ae0cbefece4
< kbn-license-sig: 58d9fcb437ac7d3ac54d538e6d5ff9a039fde738ed3a940fa530e6d8b6ef6740
< cache-control: private, no-cache, no-store, must-revalidate
< content-length: 9013976
< vary: accept-encoding
< accept-ranges: bytes
< Date: Wed, 29 Sep 2021 14:06:19 GMT
< Connection: keep-alive
< Keep-Alive: timeout=120
<
{ [13865 bytes data]
100 8802k 100 8802k 0 0 57.6M 0 --:--:-- --:--:-- --:--:-- 57.6M
* Connection #0 to host localhost left intact
Screenshot that shows that i can get data by typing url path to the browser
Update:
I try to call my function recursively. But it is bad solution even if it's working now.
case 503:
getCsvFile(urlKibanaBase, pathCsvFile)
Kibana Server needs some time to prepare response.
I've added a bit of code for signaling the channel to wait some time and after waiting repeat request to client
if res.StatusCode != 200 {
time.Sleep(30 * time.Second)
}

net.Conn never receiving EOF

I'm trying to send an HTTP request over a net.Conn TCP connection and read the subsequent response, but I never seem to receive an EOF from the connection when reading. This makes functions like io.Copy or ioutil.ReadAll useless, as they block forever.
Code:
client.go
const requestString = "GET /test HTTP/1.1\r\n" + "Host: 127.0.0.1:8080\r\n\r\n"
func main() {
dialer := net.Dialer{}
conn, err := dialer.Dial("tcp", "127.0.0.1:8080")
if err != nil {
log.Fatalln(err)
}
defer conn.Close()
_, err = conn.Write([]byte(requestString))
if err != nil {
log.Fatalln(err)
}
buf := make([]byte, 1024)
data := make([]byte, 0)
length := 0
for {
n, err := conn.Read(buf)
if err != nil {
if err != io.EOF {
fmt.Printf("Read error: %v\n", err)
}
break
}
data = append(data, buf[:n]...)
length += n
fmt.Printf("Partial read:\n%s\n", string(buf[:n]))
}
fmt.Println("Response:")
fmt.Println(string(data))
}
server.go
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Hello, %s", r.URL.Path[1:])
})
http.ListenAndServe(":8080", nil)
}
Output of running client.go with server.go already running:
Partial read:
HTTP/1.1 200 OK
Date: Wed, 25 Nov 2020 04:09:32 GMT
Content-Length: 11
Content-Type: text/plain; charset=utf-8
Hello, test
The first call to Read() returns the expected response, but lacks an EOF. The subsequent call to Read() hangs forever, and I'm not sure how to determine when the connection has finished. If I interrupt the server process, the client connection closes properly and the response is complete.
How can I either a) receive an EOF or b) determine when the response is complete? All examples I've seen online have something close to my code working, so I'm not sure what I'm doing wrong.
Adding Connection: close to the request headers makes sure that the connection... well, closes.

getting 404 in GET request in Go

When I use http://localhost:8080/login?id=ddfd#vcv.com&pwd=dccccf in postman or use it in android app I am getting 404. On curl I get
{"name":"Miss Moneypenny","email":"ddfd#vcv.com","password":"dccccf","mobile":27,"address":"dscsdacc"}
I am not able to understand what can I do to achieve json output in postman and on other platforms like Apps in ios as well as android when I use this api and also on the browser window.
My Main.go code
func getSession() *mgo.Session {
s, err := mgo.Dial("mongodb://localhost")
if err != nil {
panic(err)
}
return s
}
func main() {
r := httprouter.New()
uc := controllers.NewUserController(getSession())
r.GET("/login", uc.LoginUser)
http.ListenAndServe(":8080", r)
}
code in controller/user.go
type UserController struct {
session *mgo.Session
}
func NewUserController(s *mgo.Session) *UserController {
return &UserController{s}
}
func (uc UserController) LoginUser(w http.ResponseWriter, request *http.Request, params httprouter.Params) {
dump,err :=httputil.DumpRequest(request, true)
if err != nil {
http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
return
}
fmt.Println("Request Dump:\n", string(dump))
encodedValue := request.URL.Query().Get("id")
pwd := request.URL.Query().Get("pwd")
emailId, err := url.QueryUnescape(encodedValue)
if err != nil {
log.Fatal(err)
return
}
u := models.User{}
if err := uc.session.DB("go-web-dev-db").C("users").FindId(emailId + pwd).One(&u); err != nil {
w.WriteHeader(404)
return
}
uj, err := json.Marshal(u)
if err != nil {
fmt.Println(err)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK) // 200
fmt.Fprintf(w, "%s\n", uj)
}
code in model/user.go
type User struct {
Name string `json:"name" bson:"name"`
Email string `json:"email" bson:"_id"`
Password string `json:"password" bson:"password"`
Mobile int `json:"mobile" bson:"mobile"`
Address string `json:"address" bson:"address"`
}
After using dump when I am using i am using curl 'http://localhost:8080/login?id=ddfd#vcv.com&pwd=dccccf' I get :-
Request Dump:
GET /login?id=ddfd#vcv.com&pwd=dccccf HTTP/1.1
Host: localhost:8080
Accept: */*
User-Agent: curl/7.69.1
After using dump when I am using i am using http://localhost:8080/login?id=ddfd#vcv.com&pwd=dccccf in postman I get :-
Request Dump:
GET /login?id=ddfd#vcv.com&pwd=dccccf HTTP/1.1
Host: localhost:8080
Accept: */*
Accept-Encoding: gzip, deflate, br
Connection: keep-alive
Postman-Token: 8e925738-b8db-4656-9f53-813f4cd53a80
User-Agent: PostmanRuntime/7.24.1

Call golang jsonrpc with curl

I have "hello world" rpc service written in golang. It works fine and go jsonrpc client is working. But I need to send request with curl and this example doesn't work:
curl \
-X POST \
-H "Content-Type: application/json" \
-d '{"id": 1, "method": "Test.Say", "params": [{"greet": "world"}]}' \
http://localhost:1999/_goRPC_
Go accept connection but produce absolutely no result:
curl: (52) Empty reply from server
Here my go code:
package main
import (
"log"
"os"
"time"
"net"
"net/rpc"
"net/rpc/jsonrpc"
)
// RPC Api structure
type Test struct {}
// Greet method arguments
type GreetArgs struct {
Name string
}
// Grret message accept object with single param Name
func (test *Test) Greet(args *GreetArgs, result *string) (error) {
*result = "Hello " + args.Name
return nil
}
// Start server with Test instance as a service
func startServer(ch chan<- bool, port string) {
test := new(Test)
server := rpc.NewServer()
server.Register(test)
listener, err := net.Listen("tcp", ":" + port)
if err != nil {
log.Fatal("listen error:", err)
}
defer listener.Close()
for {
conn, err := listener.Accept()
if err != nil {
log.Fatal(err)
}
go server.ServeCodec(jsonrpc.NewServerCodec(conn))
ch <- true
}
}
// Start client and call Test.Greet method
func startClient(port string) {
conn, err := net.Dial("tcp", ":" + port)
if err != nil {
panic(err)
}
defer conn.Close()
c := jsonrpc.NewClient(conn)
var reply string
var args = GreetArgs{"world"}
err = c.Call("Test.Greet", args, &reply)
if err != nil {
log.Fatal("arith error:", err)
}
log.Println("Result: ", reply)
}
func main() {
if len(os.Args) < 2 {
log.Fatal("port not specified")
}
port := os.Args[1]
ch := make(chan bool)
go startServer(ch, port)
time.Sleep(500 * time.Millisecond)
go startClient(port)
// Produce log message each time connection closes
for {
<-ch
log.Println("Closed")
}
}
The jsonrpc package doesn't support json-rpc over HTTP currently. So, you can't call jsonrpc with curl. If you really want to do that, you can make a HTTP handler that adapts the HTTP request/response to a ServerCodec. For example:
package main
import (
"io"
"log"
"net"
"net/http"
"net/rpc"
"net/rpc/jsonrpc"
"os"
)
type HttpConn struct {
in io.Reader
out io.Writer
}
func (c *HttpConn) Read(p []byte) (n int, err error) { return c.in.Read(p) }
func (c *HttpConn) Write(d []byte) (n int, err error) { return c.out.Write(d) }
func (c *HttpConn) Close() error { return nil }
// RPC Api structure
type Test struct{}
// Greet method arguments
type GreetArgs struct {
Name string
}
// Grret message accept object with single param Name
func (test *Test) Greet(args *GreetArgs, result *string) error {
*result = "Hello " + args.Name
return nil
}
// Start server with Test instance as a service
func startServer(port string) {
test := new(Test)
server := rpc.NewServer()
server.Register(test)
listener, err := net.Listen("tcp", ":"+port)
if err != nil {
log.Fatal("listen error:", err)
}
defer listener.Close()
http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/test" {
serverCodec := jsonrpc.NewServerCodec(&HttpConn{in: r.Body, out: w})
w.Header().Set("Content-type", "application/json")
w.WriteHeader(200)
err := server.ServeRequest(serverCodec)
if err != nil {
log.Printf("Error while serving JSON request: %v", err)
http.Error(w, "Error while serving JSON request, details have been logged.", 500)
return
}
}
}))
}
func main() {
if len(os.Args) < 2 {
log.Fatal("port not specified")
}
port := os.Args[1]
startServer(port)
}
Now you can call it with curl -X POST -H "Content-Type: application/json" -d '{"id": 1, "method": "Test.Greet", "params": [{"name":"world"}]}' http://localhost:port/test
Part of the code is from this post
#jfly has a nifty solution.
Another option, if you still wanted to test with something besides the go jsonrpc cient (probably the easiest option), or use #jfly's answer, is you can use telnet to send raw data:
computer:~ User$ telnet 127.0.0.1 8888
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
{"method":"Test.Greet","params":[{"Name":"world"}],"id":0}
{"id":0,"result":"Hello world","error":null}
{"method":"Test.Greet","params":[{"Name":"world"}],"id":0}
{"id":0,"result":"Hello world","error":null}
{"method":"Test.Greet","params":[{"Name":"world"}],"id":0}
{"id":0,"result":"Hello world","error":null}
The above is the output including payload I typed in and your server's responses.
tcpdump was my friend when I was figuring out the right payload to send.

about http hijacking and keep-alive

i use
resp, err := http.Get("http://example.com/")
get a http.Response, and i want to exactly write to a http handler, but only http.ResponseWriter, so i hijack it.
...
webConn, webBuf, err := hj.Hijack()
if err != nil {
// handle error
}
defer webConn.Close()
// Write resp
resp.Write(webBuf)
...
Write raw request
But When i hijack, http connection can't reuse (keep-alive), so it slow.
How to solve?
Thanks! Sorry for my pool English.
update 12/9
keep-alive, It keep two tcp connection, and can reuse.
but when i hijack, and conn.Close(), It can't reuse old connection, so it create a new tcp connection when i each refresh.
Do not use hijack, Because once hijack, the HTTP server library will not do anything else with the connection, So can't reuse.
I change way, copy Header and Body, look like reverse proxy (http://golang.org/src/pkg/net/http/httputil/reverseproxy.go), Is works.
Example:
func copyHeader(dst, src http.Header) {
for k, w := range src {
for _, v := range w {
dst.Add(k, v)
}
}
}
func copyResponse(r *http.Response, w http.ResponseWriter) {
copyHeader(w.Header(), r.Header)
w.WriteHeader(r.StatusCode)
io.Copy(w, r.Body)
}
func handler(w http.ResponseWriter, r *http.Response) {
resp, err := http.Get("http://www.example.com")
if err != nil {
// handle error
}
copyResponse(resp, w)
}
It seem that once the connection is closed the keep-alive connection closes as well.
One possible solution would be to prevent the connection from closing until desired, but I'm not sure if that good advise.
Maybe the correct solution involves creating a instance of net.TCPConn, copying the connection over it, then calling .SetKeepAlive(true).
Before running the below example, launch another terminal with netstat -antc | grep 9090.
Routes in example:
localhost:9090/ok is a basic (non-hijacked) connection
localhost:9090 is a hijacked connection, lasting for 10 seconds.
Example
package main
import (
"fmt"
"net/http"
"sync"
"time"
)
func checkError(e error) {
if e != nil {
panic(e)
}
}
var ka_seconds = 10
var conn_id = 0
func main() {
http.HandleFunc("/ok", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "ok")
})
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
conn_id++
fmt.Printf("Connection %v: Keep-alive is enabled %v seconds\n", conn_id, ka_seconds)
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError)
return
}
conn, bufrw, err := hj.Hijack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Don't forget to close the connection:
time.AfterFunc(time.Second* time.Duration(ka_seconds), func() {
conn.Close()
fmt.Printf("Connection %v: Keep-alive is disabled.\n", conn_id)
})
resp, err := http.Get("http://www.example.com")
checkError(err)
resp.Write(bufrw)
bufrw.Flush()
})
fmt.Println("Listing to localhost:9090")
http.ListenAndServe(":9090", nil)
}
Related issue: http://code.google.com/p/go/issues/detail?id=5645

Resources