I have the following:
//In an init func
if logStashHost != "" {
lsconn, err = net.Dial("tcp", logStashHost)
}
...
ToLogStash(rec, lsconn)
Then Two functions:
func ReadLogStash(conn net.Conn) {
buffer := make([]byte, 256)
for {
_, err := conn.Read(buffer)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(buffer)
}
}
}
func ToLogStash(r *logrow.Record, conn net.Conn) {
b, err := json.Marshal(r)
if err != nil {
fmt.Println(err)
return
}
_, err = fmt.Fprintln(conn, string(b))
if err != nil {
fmt.Println(err)
}
}
Where ReadLogStash is a running goroutine. If the other side closes, I get EOF. What would be a good implementation in ReadLogStash to have it attempt to reestablish the connection every X seconds when it gets an EOF?
Go has channels for synchronization and communication, use them!
Make your connection in a loop, and have it wait for some sort of message to come back on a channel.
...
errCh := make(chan error)
for {
lsconn, err = net.Dial("tcp", logStashHost)
// check error!
go ReadLogStash(lsconn, errCh)
err = <-errCh
if err != nil {
// bad error
break
}
// sleep to backoff on retries?
}
...
func ReadLogStash(conn net.Conn, errCh chan error) {
_, err := io.Copy(os.Stderr, conn)
if err != nil {
fmt.Println(err)
}
// a nil error from io.Copy means you reached EOF.
errCh <- err
}
Unless you have more functionality in ReadLogStash, you can probably just use io.Copy inline, and forget the entire function, but this pattern may come in useful for you anyway.
Here is what I ended up going with, a channel was the right direction:
if logStashHost != "" {
lsc = make(chan *logrow.Record)
go ToLogStash(lsc, logStashHost)
}
...
if lsc != nil {
lsc <- rec
}
...
func ToLogStash(c chan *logrow.Record, logStashHost string) {
var lsconn net.Conn
var enc *json.Encoder
var err error
connect := func() {
for {
lsconn, err = net.Dial("tcp", logStashHost)
if err == nil {
enc = json.NewEncoder(lsconn)
break
}
log.Println(err)
time.Sleep(time.Second)
}
}
connect()
for r := range c {
err = enc.Encode(r)
if err != nil {
lsconn.Close()
log.Println(err)
connect()
}
}
}
Related
Supposed that I'm writing an http handler, that do something else before returning a response, do I have to setup a listener to check wether the http request context has been canceled? so that it can return immediately, or is there any other way to exit the handler when the request context cancelled?
func handleSomething(w http.ResponseWriter, r *http.Request) {
done := make(chan error)
go func() {
if err := doSomething(r.Context()); err != nil {
done <- err
return
}
done <- nil
}()
select {
case <-r.Context().Done():
http.Error(w, r.Context().Err().Error(), http.StatusInternalServerError)
return
case err := <-done:
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}
}
func doSomething(ctx context.Context) error {
// simulate doing something for 1 second.
time.Sleep(time.Second)
return nil
}
I tried making a test for it, but after the context got cancelled, doSomething function didn't stop and still running in the background.
func TestHandler(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/something", handleSomething)
srv := http.Server{
Addr: ":8989",
Handler: mux,
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.ListenAndServe(); err != nil {
log.Println(err)
}
}()
time.Sleep(time.Second)
req, err := http.NewRequest(http.MethodGet, "http://localhost:8989/something", nil)
if err != nil {
t.Fatal(err)
}
cl := http.Client{
Timeout: 3 * time.Second,
}
res, err := cl.Do(req)
if err != nil {
t.Logf("error: %s", err.Error())
} else {
t.Logf("request is done with status code %d", res.StatusCode)
}
go func() {
<-time.After(10 * time.Second)
shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
srv.Shutdown(shutdown)
}()
wg.Wait()
}
func handleSomething(w http.ResponseWriter, r *http.Request) {
done := make(chan error)
go func() {
if err := doSomething(r.Context()); err != nil {
log.Println(err)
done <- err
}
done <- nil
}()
select {
case <-r.Context().Done():
log.Println("context is done!")
return
case err := <-done:
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}
}
func doSomething(ctx context.Context) error {
return runInContext(ctx, func() {
log.Println("doing something")
defer log.Println("done doing something")
time.Sleep(10 * time.Second)
})
}
func runInContext(ctx context.Context, fn func()) error {
ch := make(chan struct{})
go func() {
defer close(ch)
fn()
}()
select {
case <-ctx.Done():
return ctx.Err()
case <-ch:
return nil
}
}
I've just refactored the solution provided a little bit and now it should work. Let me guide you through the relevant changes.
The doSomething function
func doSomething(ctx context.Context) error {
fmt.Printf("%v - doSomething: start\n", time.Now())
select {
case <-ctx.Done():
fmt.Printf("%v - doSomething: cancelled\n", time.Now())
return ctx.Err()
case <-time.After(3 * time.Second):
fmt.Printf("%v - doSomething: processed\n", time.Now())
return nil
}
}
It waits for a cancellation input or after a delay of 3 seconds it returns to the caller. It accepts a context to listen for.
The handleSomething function
func handleSomething(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
fmt.Printf("%v - handleRequestCtx: start\n", time.Now())
done := make(chan error)
go func() {
if err := doSomething(ctx); err != nil {
fmt.Printf("%v - handleRequestCtx: error %v\n", time.Now(), err)
done <- err
}
done <- nil
}()
select {
case <-ctx.Done():
fmt.Printf("%v - handleRequestCtx: cancelled\n", time.Now())
return
case err := <-done:
if err != nil {
fmt.Printf("%v - handleRequestCtx: error: %v\n", time.Now(), err)
w.WriteHeader(http.StatusInternalServerError)
return
}
fmt.Printf("%v - handleRequestCtx: processed\n", time.Now())
}
}
Here, the logic is very similar to yours. In the select, we check whether the received error is nil or not, and based on this we return to the proper HTTP status code to the caller. If we receive a cancellation input, we cancel all the context chain.
The TestHandler function
func TestHandler(t *testing.T) {
r := mux.NewRouter()
r.HandleFunc("/demo", handleSomething)
srv := http.Server{
Addr: ":8000",
Handler: r,
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
if err := srv.ListenAndServe(); err != nil {
fmt.Println(err.Error())
}
}()
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 1*time.Second) // request canceled
// ctx, cancel := context.WithTimeout(ctx, 5*time.Second) // request processed
defer cancel()
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8000/demo", nil)
client := http.Client{}
res, err := client.Do(req)
if err != nil {
fmt.Println(err.Error())
} else {
fmt.Printf("res status code: %d\n", res.StatusCode)
}
srv.Shutdown(ctx)
wg.Wait()
}
Here, we spin up an HTTP server and issue an HTTP request to it through an http.Client. You can see that there are two statements to set the context timeout. If you use the one with the comment // request canceled, everything will be canceled, otherwise, if you use the other the request will be processed.
I Hope that this clarifies your question!
trying to post multipart/form-data image using go
image file receive from request client and already saved as multipart.File
here my code
func postImage(file multipart.File, url string, filename string) (*http.Response, error) {
r, w := io.Pipe()
defer w.Close()
m := multipart.NewWriter(w)
defer m.Close()
errchan := make(chan error)
defer close(errchan)
go func() {
part, err := m.CreateFormFile("file", filename)
log.Println(err)
if err != nil {
errchan <- err
return
}
if _, err := io.Copy(part, file); err != nil {
errchan <- err
return
}
}()
merr := <-errchan
if merr != nil {
return nil, merr
}
resp, err := http.Post(url, m.FormDataContentType(), r)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return resp, err
}
when i try using it, it stuck at part, err := m.CreateFormFile("file", filename) never return anything
any solution?
Thanks
Use the pipe error to propagate the error back to the main goroutine. Close the write side of the pipe to prevent the client from blocking forever on read. Close the read side of the pipe to ensure that the goroutine exits.
func postImage(file multipart.File, url string, filename string) (*http.Response, error) {
r, w := io.Pipe()
// Close the read side of the pipe to ensure that
// the goroutine exits in the case where http.Post
// does not read all of the request body.
defer r.Close()
m := multipart.NewWriter(w)
go func() {
part, err := m.CreateFormFile("file", filename)
if err != nil {
// The error is returned from read on the pipe.
w.CloseWithError(err)
return
}
if _, err := io.Copy(part, file); err != nil {
// The error is returned from read on the pipe.
w.CloseWithError(err)
return
}
// The http.Post function reads the pipe until
// an error or EOF. Close to return an EOF to
// http.Post.
w.Close()
}()
resp, err := http.Post(url, m.FormDataContentType(), r)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return resp, err
}
I am having issue in keeping websocket connection alive in go. In my code below, I assign 2 different ports to handle websocket (:8080) and for API request (:3300).
There is no issue when I am using websocket handler directly, but using API handler request and making new external request to the websocker handler, the connection closed directly. Any help is appreciated.
package main
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/gobwas/ws"
"github.com/gobwas/ws/wsutil"
)
func main() {
go websocket()
http.HandleFunc("/ws", func(rw http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
url := fmt.Sprintf("http://127.0.0.1:8080?%s", r.URL.RawQuery)
req, err := http.NewRequest(r.Method, url, bytes.NewReader(body))
if err != nil {
fmt.Println(err)
panic(err)
}
req.Header = make(http.Header)
for h, val := range r.Header {
req.Header[h] = val
}
httpClient := &http.Client{Timeout: time.Second * 10}
httpClient.Do(req)
})
http.ListenAndServe(":3300", nil)
}
func websocket() {
http.ListenAndServe(":8080", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, _, _, err := ws.UpgradeHTTP(r, w)
if err != nil {
fmt.Println(err)
return
}
go func() {
defer conn.Close()
for {
msg, op, err := wsutil.ReadClientData(conn)
if err != nil {
fmt.Println(err)
return
}
err = wsutil.WriteServerMessage(conn, op, msg)
if err != nil {
fmt.Println(err)
return
}
}
}()
}))
}
The code in the question connects to the websocket endpoint using an HTTP request. Upgrade fails as a result.
Use the standard library reverse proxy to proxy the request.
A simpler approach is to is to call the websocket handler directly. Move the handler to a top-level function:
func handleWS(w http.ResponseWriter, r *http.Request) {
conn, _, _, err := ws.UpgradeHTTP(r, w)
if err != nil {
fmt.Println(err)
return
}
go func() {
defer conn.Close()
for {
msg, op, err := wsutil.ReadClientData(conn)
if err != nil {
fmt.Println(err)
return
}
err = wsutil.WriteServerMessage(conn, op, msg)
if err != nil {
fmt.Println(err)
return
}
}
}()
}
Use the handler in both servers.
func main() {
go websocket()
http.HandleFunc("/ws", handleWS)
http.ListenAndServe(":3300", nil)
}
func websocket() {
http.ListenAndServe(":8080", http.HandlerFunc(handleWS))
}
I have a long running app that I'd like to monitor in real time. HAR files allow me to do this after the fact, but as they are an "archive", they don't allow me to do this in real time.
Is their anyway to stream the "events" array of the HAR file so I can process them as they are generated?
This can be firefox or chrome.
So with some help from https://github.com/mafredri/cdp/tree/master/example/screencast I figured out how to do this in go with chrome's debugger api
What this code doesn't do is tie the request body to the response (where it isn't available), but as I show the RequestID will be consistent so if one serializes event processing (say via locking) one can save the body and use it when the response event is seen.
package main
import (
"context"
"log"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/cdpcmd"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/rpcc"
)
func main() {
if err := run(); err != nil {
panic(err)
}
}
func run() error {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
devt := devtool.New("http://localhost:9222")
page, err := devt.Get(ctx, devtool.Page)
if err != nil {
return err
}
conn, err := rpcc.DialContext(ctx, page.WebSocketDebuggerURL)
if err != nil {
return err
}
defer conn.Close()
c := cdp.NewClient(conn)
err = c.Page.Enable(ctx)
if err != nil {
return err
}
loadEventFired, err := c.Page.LoadEventFired(ctx)
if err != nil {
return err
}
_, err = c.Page.Navigate(ctx, cdpcmd.NewPageNavigateArgs("https://github.com/"))
if err != nil {
return err
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
loadEventFired.Close()
a := &cdpcmd.NetworkEnableArgs{}
a.SetMaxResourceBufferSize(32000)
a.SetMaxTotalBufferSize(96000)
err = c.Network.Enable(ctx, a)
responseEvents, err := c.Network.ResponseReceived(ctx)
requestEvents, err := c.Network.RequestWillBeSent(ctx)
go func() {
defer responseEvents.Close()
for {
ev, err := responseEvents.Recv()
if err != nil {
log.Printf("Failed to receive network event: %v", err)
return
}
log.Printf("requestid = %v, url = %v", ev.RequestID, ev.Response.URL)
}
}()
go func() {
defer requestEvents.Close()
for {
ev, err := requestEvents.Recv()
if err != nil {
log.Printf("Failed to receive network event: %v", err)
return
}
log.Printf("requestid = %v, url = %v", ev.RequestID, ev.Request.URL)
}
}()
select {}
return nil
}
I am trying to find the list of all directories using a recursive function. The code to the function is
func FindDirs(dir string, nativePartitions []int64, wg *sync.WaitGroup, dirlistchan chan string) {
// defer wg.Done here will give negative waitgroup panic, commenting it will give negative waitgroup counter panic
fd, err := os.Open(dir)
if err != nil {
panic(err)
}
filenames, err := fd.Readdir(0)
if err != nil {
panic(err)
}
for _, i := range filenames {
var buff bytes.Buffer
buff.WriteString(dir)
switch dir {
case "/":
default:
buff.WriteString("/")
}
buff.WriteString(i.Name())
/*err := os.Chdir(dir)
if err != nil {
return err
}*/
t := new(syscall.Statfs_t)
err = syscall.Statfs(buff.String(), t)
if err != nil {
//fmt.Println("Error accessing", buff.String())
}
if checkDirIsNative(t.Type, nativePartitions) && i.IsDir(){
dirlistchan <- buff.String()
FindDirs(buff.String(), nativePartitions, wg, dirlistchan) //recursion happens here
} else {
//fmt.Println(i.Name(), "is not native")
}
}
}
and in the main function, I am calling it as
wg := new(sync.WaitGroup)
dirlistchan := make(chan string, 1000)
wg.Add(1)
go func() {
filtermounts.FindDirs(parsedConfig.ScanFrom, []int64{filtermounts.EXT4_SUPER_MAGIC}, wg, dirlistchan)
}()
go func() {
wg.Wait()
close(dirlistchan)
}()
for i := range dirlistchan {
fmt.Println(i)
}
wg.Wait()
and I am getting a
fatal error: all goroutines are asleep - deadlock!
I was able to get this working if I am printing the result instead of using channels, or append to a slice using mutex. (verified with the linux find command to see if the results are same.) Please find the function after omitting channels and using sync.Mutex and append.
func FindDirs(dir string, nativePartitions []int64, dirlist *[]string, mutex *sync.Mutex) []string{
fd, err := os.Open(dir)
defer fd.Close()
if err != nil {
panic(err)
}
filenames, err := fd.Readdir(0)
if err != nil {
panic(err)
}
for _, i := range filenames {
var buff bytes.Buffer
buff.WriteString(dir)
switch dir {
case "/":
default:
buff.WriteString("/")
}
buff.WriteString(i.Name())
/*err := os.Chdir(dir)
if err != nil {
return err
}*/
t := new(syscall.Statfs_t)
err = syscall.Statfs(buff.String(), t)
if err != nil {
//fmt.Println("Error accessing", buff.String())
}
if checkDirIsNative(t.Type, nativePartitions) && i.IsDir(){
//dirlistchan <- buff.String()
mutex.Lock()
*dirlist = append(*dirlist, buff.String())
mutex.Unlock()
//fmt.Println(buff.String())
FindDirs(buff.String(), nativePartitions, dirlist, mutex)
} else {
//fmt.Println(i.Name(), "is not native")
}
}
return *dirlist
}
But I cannot think of a way to make this work with channels and goroutines. Any help is greatly appreciated.
Note: Here is a link to the golang playground with the code. I couldn't find a workaround to get the syscall thing to work on the playground either. It works on my system though.
Thanks.
Short answer : You are not closing the channel.
Fix : add defer wg.Done() at beginning of the go routine that calls FindDirs
go func() {
defer wg.Done()
filtermounts.FindDirs(parsedConfig.ScanFrom, []int64{filtermounts.EXT4_SUPER_MAGIC}, wg, dirlistchan)
}()
Why did it happen
The go routine that is responsponsible for closing the channel waits for wg there is no wg.Done in the code above. So close never happens
Now the for loop blocks on the channel for close or a value for ever, this cause the error
fatal error: all goroutines are asleep - deadlock!
So here is your code ,this may be run as
go run filename.go /path/to/folder
Code
package main
import (
"bytes"
"fmt"
"os"
"sync"
"syscall"
)
func main() {
wg := new(sync.WaitGroup)
dirlistchan := make(chan string, 1000)
wg.Add(1)
go func() {
defer wg.Done()
FindDirs(os.Args[1], []int64{61267}, wg, dirlistchan)
}()
go func() {
wg.Wait()
close(dirlistchan)
}()
for i := range dirlistchan {
fmt.Println(i)
}
wg.Wait()
}
func FindDirs(dir string, nativePartitions []int64, wg *sync.WaitGroup, dirlistchan chan string) {
fd, err := os.Open(dir)
if err != nil {
panic(err)
}
filenames, err := fd.Readdir(0)
if err != nil {
panic(err)
}
for _, i := range filenames {
var buff bytes.Buffer
buff.WriteString(dir)
switch dir {
case "/":
default:
buff.WriteString("/")
}
buff.WriteString(i.Name())
/*err := os.Chdir(dir)
if err != nil {
return err
}*/
t := new(syscall.Statfs_t)
err = syscall.Statfs(buff.String(), t)
if err != nil {
//fmt.Println("Error accessing", buff.String())
}
if checkDirIsNative(t.Type, nativePartitions) && i.IsDir() {
dirlistchan <- buff.String()
FindDirs(buff.String(), nativePartitions, wg, dirlistchan) //recursion happens here
} else {
//fmt.Println(i.Name(), "is not native")
}
}
}
func checkDirIsNative(dirtype int64, nativetypes []int64) bool {
for _, i := range nativetypes {
if dirtype == i {
return true
}
}
return false
}
Find the go.play link here
As has been stated already you should close the channel if you want the main goroutine to exit.
Example of implementation :
In function func FindDirs you could make an additional channel for every recursive func FindDirs call that this function is going to make and pass that new channel in the argument. Then simultaneously listen to all those new channels and forward the strings back to the channel that function got in the argument.
After all new channels has been closed close the channel given in the argument.
In other words every func call should have its own channel that it sends to. The string is then forwarded all the way to main function.
Dynamic select described here : how to listen to N channels? (dynamic select statement)