Troubles on constructing HLS using libp2p-http - http

I intend to implement a classic HLS example with libp2p, which is as follows
https://hackernoon.com/building-a-media-streaming-server-using-go-and-hls-protocol-j85h3wem
But I encountered the following errors:
404 page not found
My code constructing linstening:
m1, _ := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/1000")
m2, _ := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/1001")
srvHost := NewHost(m1)
clientHost := NewHost(m2)
defer srvHost.Close()
defer clientHost.Close()
srvHost.Peerstore().AddAddrs(clientHost.ID(), clientHost.Addrs(), peerstore.PermanentAddrTTL)
fmt.Println("id is", clientHost.ID())
fmt.Println("addr is", srvHost.Addrs())
clientHost.Peerstore().AddAddrs(srvHost.ID(), srvHost.Addrs(), peerstore.PermanentAddrTTL)
listener, err := gostream.Listen(srvHost, "/testiti-test")
if err != nil {
panic(err)
}
defer listener.Close()
My code constructing client:
tr := &http.Transport{}
tr.RegisterProtocol("libp2p", p2phttp.NewTransport(clientHost, p2phttp.ProtocolOption("/testiti-test")))
client := &http.Client{Transport: tr}
res, err := client.Get(fmt.Sprintf("libp2p://%s/simple", port))
if err != nil {
panic(err)
}
data, err := ioutil.ReadAll(res.Body)
defer res.Body.Close()
if err != nil {
log.Fatal(err)
}
fmt.Println(string(data))
My code constructing sever:
go func() {
http.HandleFunc("/simple", addHeaders(http.FileServer(http.Dir(songsDir))))
//http.HandleFunc("/simple", addHeaders2())
fmt.Printf("Starting server on %v\n", port)
log.Printf("Serving %s on HTTP port: %v\n", songsDir, port)
server := &http.Server{}
server.Serve(listener)
}()
func addHeaders(h http.Handler) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
h.ServeHTTP(w, r)
}
}
New host:
func NewHost(listen multiaddr.Multiaddr) host.Host {
h, err := libp2p.New(
libp2p.ListenAddrs(listen),
)
if err != nil {
panic(err)
}
return h
}

The problem is casued by routing.For a http.FileServer,"\simple" means a dir but not a url.So just fix like this:
url := fmt.Sprintf("libp2p://%s/origin.m3u8", port)
res, err := client.Get(url)
if err != nil {
panic(err)
}
data, err := ioutil.ReadAll(res.Body)
defer res.Body.Close()
if err != nil {
log.Fatal(err)
}
fmt.Println(string(data))

Related

Keep WebSocket connection alive after upgrade in Go

I am having issue in keeping websocket connection alive in go. In my code below, I assign 2 different ports to handle websocket (:8080) and for API request (:3300).
There is no issue when I am using websocket handler directly, but using API handler request and making new external request to the websocker handler, the connection closed directly. Any help is appreciated.
package main
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/gobwas/ws"
"github.com/gobwas/ws/wsutil"
)
func main() {
go websocket()
http.HandleFunc("/ws", func(rw http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
url := fmt.Sprintf("http://127.0.0.1:8080?%s", r.URL.RawQuery)
req, err := http.NewRequest(r.Method, url, bytes.NewReader(body))
if err != nil {
fmt.Println(err)
panic(err)
}
req.Header = make(http.Header)
for h, val := range r.Header {
req.Header[h] = val
}
httpClient := &http.Client{Timeout: time.Second * 10}
httpClient.Do(req)
})
http.ListenAndServe(":3300", nil)
}
func websocket() {
http.ListenAndServe(":8080", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, _, _, err := ws.UpgradeHTTP(r, w)
if err != nil {
fmt.Println(err)
return
}
go func() {
defer conn.Close()
for {
msg, op, err := wsutil.ReadClientData(conn)
if err != nil {
fmt.Println(err)
return
}
err = wsutil.WriteServerMessage(conn, op, msg)
if err != nil {
fmt.Println(err)
return
}
}
}()
}))
}
The code in the question connects to the websocket endpoint using an HTTP request. Upgrade fails as a result.
Use the standard library reverse proxy to proxy the request.
A simpler approach is to is to call the websocket handler directly. Move the handler to a top-level function:
func handleWS(w http.ResponseWriter, r *http.Request) {
conn, _, _, err := ws.UpgradeHTTP(r, w)
if err != nil {
fmt.Println(err)
return
}
go func() {
defer conn.Close()
for {
msg, op, err := wsutil.ReadClientData(conn)
if err != nil {
fmt.Println(err)
return
}
err = wsutil.WriteServerMessage(conn, op, msg)
if err != nil {
fmt.Println(err)
return
}
}
}()
}
Use the handler in both servers.
func main() {
go websocket()
http.HandleFunc("/ws", handleWS)
http.ListenAndServe(":3300", nil)
}
func websocket() {
http.ListenAndServe(":8080", http.HandlerFunc(handleWS))
}

Stream HAR events

I have a long running app that I'd like to monitor in real time. HAR files allow me to do this after the fact, but as they are an "archive", they don't allow me to do this in real time.
Is their anyway to stream the "events" array of the HAR file so I can process them as they are generated?
This can be firefox or chrome.
So with some help from https://github.com/mafredri/cdp/tree/master/example/screencast I figured out how to do this in go with chrome's debugger api
What this code doesn't do is tie the request body to the response (where it isn't available), but as I show the RequestID will be consistent so if one serializes event processing (say via locking) one can save the body and use it when the response event is seen.
package main
import (
"context"
"log"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/cdpcmd"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/rpcc"
)
func main() {
if err := run(); err != nil {
panic(err)
}
}
func run() error {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
devt := devtool.New("http://localhost:9222")
page, err := devt.Get(ctx, devtool.Page)
if err != nil {
return err
}
conn, err := rpcc.DialContext(ctx, page.WebSocketDebuggerURL)
if err != nil {
return err
}
defer conn.Close()
c := cdp.NewClient(conn)
err = c.Page.Enable(ctx)
if err != nil {
return err
}
loadEventFired, err := c.Page.LoadEventFired(ctx)
if err != nil {
return err
}
_, err = c.Page.Navigate(ctx, cdpcmd.NewPageNavigateArgs("https://github.com/"))
if err != nil {
return err
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
loadEventFired.Close()
a := &cdpcmd.NetworkEnableArgs{}
a.SetMaxResourceBufferSize(32000)
a.SetMaxTotalBufferSize(96000)
err = c.Network.Enable(ctx, a)
responseEvents, err := c.Network.ResponseReceived(ctx)
requestEvents, err := c.Network.RequestWillBeSent(ctx)
go func() {
defer responseEvents.Close()
for {
ev, err := responseEvents.Recv()
if err != nil {
log.Printf("Failed to receive network event: %v", err)
return
}
log.Printf("requestid = %v, url = %v", ev.RequestID, ev.Response.URL)
}
}()
go func() {
defer requestEvents.Close()
for {
ev, err := requestEvents.Recv()
if err != nil {
log.Printf("Failed to receive network event: %v", err)
return
}
log.Printf("requestid = %v, url = %v", ev.RequestID, ev.Request.URL)
}
}()
select {}
return nil
}

Can I post with Content-Type: multipart/form-data

How do I POST to an API with Content-Type: multipart/form-data, []byte parameters and string arguments? I have tried, but it is failing.
Error message:
details: "[301 301 Moved Permanently]<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML 2.0//EN\">\r\n<html>\r\n301 Moved Permanently\r\n<body bgcolor=\"white\">\r\n301 Moved Permanently\r\n<p>The requested resource has been assigned a new permanent URI.</p >\r\n<hr/>Powered by Tengine/2.1.0</body>\r\n</html>\r\n"
Go code:
func NewPost2(url string) ([]byte, error) {
m := make(map[string]interface{}, 0)
m["fileName"] ="good"
m["name"] = Base64ToByte("/9j/4AAQSkZJRgABAQEAeAB4AAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAABAAEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDHooor+wD+Zz//2Q==")
b, _ := json.Marshal(m)
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
httpReq.Header.Set("Content-Type", "multipart/form-data;charset=UTF-8")
client := &http.Client{}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
b, _ := ioutil.ReadAll(resp.Body)
return nil, fmt.Errorf("[%d %s]%s", resp.StatusCode, resp.Status, string(b))
}
respData, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return respData, nil
}
Now, I am very happy with the mood to share my solution
func NewPostFile(url string, paramTexts map[string]interface{}, paramFile FileItem) ([]byte, error) {
// if paramFiles ==nil {
// return NewPost(url,paramTexts,header,transport)
// }
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
for k, v := range paramTexts {
bodyWriter.WriteField(k, v.(string))
}
fileWriter, err := bodyWriter.CreateFormFile(paramFile.Key, paramFile.FileName)
if err != nil {
fmt.Println(err)
//fmt.Println("Create form file error: ", error)
return nil, err
}
fileWriter.Write(paramFile.Content)
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
fmt.Println(bodyBuf.String())
resp, err := http.Post(url, contentType, bodyBuf)
if err != nil {
return nil, err
}
defer resp.Body.Close()
fmt.Println(resp)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
b, _ := ioutil.ReadAll(resp.Body)
return nil, fmt.Errorf("[%d %s]%s", resp.StatusCode, resp.Status, string(b))
}
respData, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
fmt.Println(string(respData))
return respData, nil
}
type FileItem struct {
Key string //image_content
FileName string //test.jpg
Content []byte //[]byte
}
I wrapped the multipart code in a function, as you need to Close it before you
can make a request. Also my method is using # as a heuristic, similar to
cURL [1]:
package main
import (
"bytes"
"io"
"mime/multipart"
"os"
"strings"
)
func createForm(form map[string]string) (string, io.Reader, error) {
body := new(bytes.Buffer)
mp := multipart.NewWriter(body)
defer mp.Close()
for key, val := range form {
if strings.HasPrefix(val, "#") {
val = val[1:]
file, err := os.Open(val)
if err != nil { return "", nil, err }
defer file.Close()
part, err := mp.CreateFormFile(key, val)
if err != nil { return "", nil, err }
io.Copy(part, file)
} else {
mp.WriteField(key, val)
}
}
return mp.FormDataContentType(), body, nil
}
Example:
package main
import "net/http"
func main() {
form := map[string]string{"profile": "#portrait.jpg", "name": "John"}
ct, body, err := createForm(form)
if err != nil {
panic(err)
}
http.Post("https://stackoverflow.com", ct, body)
}
https://curl.se/docs/manpage.html#-F
On a 301 response, the new url is specified in the headers of the response, not its body
(see https://en.wikipedia.org/wiki/HTTP_301)
try printing :
resp.Header["Location"]
If you have this error as a final response, this also means that the http.Client chose to not follow this redirection.
The doc says that the dafult policy for a Client is to follow up to 10 redirects.
In order to debug redirects, you can write your own CheckRedirect function, which can for instance print the sequence of urls

golang and ListenAndServeTLS return error

func main(){
runtime.GOMAXPROCS(runtime.NumCPU())
r := mux.NewRouter()
www := r.Host("secure.domain.com").Subrouter()
www.HandleFunc("/", Handler_www)
api := r.Host("api.domain.com").Subrouter()
api.HandleFunc("/", Handler_api)
go func(){
cert := "/var/ini/ssl/secure.domain.com/intermediate.crt"
prv_key := "/var/ini/ssl/secure.domain.com/private.key"
err := http.ListenAndServeTLS(":9443", cert, prv_key, r)
if err != nil {
log.Fatal("HTTPS server: ", err)
}
}
err := http.ListenAndServe(":9000", r)
if err != nil {
log.Fatal("HTTP server: ", err)
}
}
Try and add () after the closing }.
That way, you would actually call the func().
go func(){
cert := "/var/ini/ssl/secure.domain.com/intermediate.crt"
prv_key := "/var/ini/ssl/secure.domain.com/private.key"
err := http.ListenAndServeTLS(":9443", cert, prv_key, r)
if err != nil {
log.Fatal("HTTPS server: ", err)
}
}()

How to make a long connection with http.Client?

I try to connect a http server as long connection, like below:
func main() {
request, err := http.NewRequest("GET", "http://long.connection.org:8080/", nil)
request.SetBasicAuth("xxx", "oooo")
http_client := &http.Client{}
response, _ := http_client.Do(request)
var buf []byte
for {
_, err := response.Body.Read(buf)
if err == io.EOF { break }
fmt.Printf("%s", string(buf))
}
}
But read from response.Body always empty. And seems I can't use response.Body to send data to server.
Any one can help?
This seems to work:
package main
import (
"fmt"
"io"
"log"
"net/http"
)
func main() {
request, err := http.NewRequest("GET", "http://www.example.com/", nil)
if err != nil {
log.Fatal(err)
}
http_client := &http.Client{}
response, err := http_client.Do(request)
if err != nil {
log.Fatal(err)
}
buf := make([]byte, 4096) // any non zero value will do, try '1'.
for {
n, err := response.Body.Read(buf)
if n == 0 && err != nil { // simplified
break
}
fmt.Printf("%s", buf[:n]) // no need to convert to string here
}
fmt.Println()
}
Edit: Added forgotten error handling of NewRequest.

Resources