Golang FileServer check progress - http

I have created a simple FileServer using GoLang
package main
import (
"log"
"net/http"
)
func loggingHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Println(r.Method, r.RemoteAddr,r.URL.Path)
h.ServeHTTP(w, r)
})
}
func main() {
http.ListenAndServe(":8080", loggingHandler(http.FileServer(http.Dir("/directory/subdir/"))))
}
Now I am looking for a way to check if file is properly downloaded by host, or monitor how many bytes are arleady downloaded.
Unfortunetly I am unable to find any hints on the internet or if it is even posibble.
Has anyone encountered a similar problem? Is it possible to verify the correctness of downloading the file from the server?

You can provide your implementation of FileSystem with a progress counter to the FileServer.
Example:
package main
import (
"net/http"
)
type fileWithProgress struct {
http.File
size int64
downloaded int64
lastReported int64
name string
client string
}
func (f *fileWithProgress) Read(p []byte) (n int, err error) {
n, err = f.File.Read(p)
f.downloaded += int64(n)
f.lastReported += int64(n)
if f.lastReported > 1_000_000 { // report every 1e6 bytes
println("client:", f.client, "file:", f.name, "downloaded:", f.downloaded*100/f.size, "%")
f.lastReported = 0
}
return n, err
}
func (f *fileWithProgress) Seek(offset int64, whence int) (int64, error) {
f.downloaded = 0 // reset content detection read if no content-type specified
return f.File.Seek(offset, whence)
}
func (f *fileWithProgress) Close() error {
println("client:", f.client, "file:", f.name, "done", f.downloaded, "of", f.size, "bytes")
return f.File.Close()
}
type dirWithProgress struct {
http.FileSystem
client string
}
func (d *dirWithProgress) Open(name string) (http.File, error) {
println("open:", name, "for the client:", d.client)
f, err := d.FileSystem.Open(name)
if err != nil {
return nil, err
}
st, err := f.Stat()
if err != nil {
return nil, err
}
return &fileWithProgress{File: f, size: st.Size(), name: st.Name(), client: d.client}, nil
}
func fileServer(fs http.FileSystem) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) {
progressDir := dirWithProgress{FileSystem: fs, client: r.RemoteAddr}
http.FileServer(&progressDir).ServeHTTP(w, r)
}
return http.HandlerFunc(f)
}
func main() {
dir := http.Dir(".")
fs := fileServer(dir)
http.Handle("/", fs)
http.ListenAndServe("127.0.0.1:8080", nil)
}

Related

Golang mux http.FileServer not returning image

I am new to go and am trying to setup a go server. My intention is to return an image when the url is hit.
this is what i have done
myRouter := mux.NewRouter()
myRouter.HandleFunc("/poster_path/{id}",posterfunc)
This is my posterfunc
func posterfunc(w http.ResponseWriter, r *http.Request){
w.Header().Set("Content-Type", "image/jpeg")
vars := mux.Vars(r)
key := vars["id"]
var url = "/home/rakshithjk/Desktop/poster/"+key+".jpg"
http.FileServer(http.Dir(url))
}
This is the output in Postman -
Any help would be appreciated.
UPDATE -
tried changing http.FileServer to http.ServeFile, but the output remains the same
Modified handler function
func posterfunc(w http.ResponseWriter, r *http.Request){
w.Header().Set("Content-Type", "image/jpeg")
vars := mux.Vars(r)
key := vars["id"]
var url = "/home/rakshithjk/Desktop/poster/"+key+".jpg"
http.ServeFile(w, r,url)
This is my entire file contents(for reference)
package main
import (
"fmt"
"log"
"net/http"
"encoding/json"
"database/sql"
_ "github.com/go-sql-driver/mysql"
"github.com/gorilla/mux"
//"github.com/davecgh/go-spew/spew"
)
func handleRequests() {
myRouter := mux.NewRouter()
myRouter.HandleFunc("/", homePage)
myRouter.HandleFunc("/movie/top_rated", returnSingleArticle)
myRouter.HandleFunc("/poster_path",posterfunc)
log.Fatal(http.ListenAndServe(":10000", myRouter))
}
func enableCors(w *http.ResponseWriter) {
(*w).Header().Set("Access-Control-Allow-Origin", "*")
}
type movie_list struct {
Page int `json:"page"`
Results []movie `json:"results"`
}
type movie struct {
Id int `json:"id"`
Title string `json:"title"`
Language string `json:"language"`
Release_date string `json:"release_date"`
Poster_path string `json:"poster_path"`
Background_path string `json:"background_path"`
Overview string `json:"overview"`
Genre_ids string `json:"genre_ids"`
}
func posterfunc(w http.ResponseWriter, r *http.Request){
w.Header().Set("Content-Type", "image/jpeg")
//vars := mux.Vars(r)
//key := vars["id"]
enableCors(&w)
var url = "/home/rakshithjk/go/src/clumio/112.png"
fmt.Fprintf(w, "Hello, %q\n", url)
http.ServeFile(w, r,url)
}
func homePage(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Welcome to the HomePage!")
fmt.Println("Endpoint Hit: homePage")
}
func returnSingleArticle(w http.ResponseWriter, r *http.Request) {
//vars := mux.Vars(r)
//key := vars["id"]
enableCors(&w)
db, err := sql.Open("mysql", "root:72574484#tcp(127.0.0.1:3306)/PicturePerfect")
if err != nil {
fmt.Println(err)
}else{
fmt.Println("Connection Established")
}
rows,err:=db.Query("select * from movies limit 10")
if err!=nil{
fmt.Println(err)
}
var list movie_list
var tag movie
for rows.Next(){
err:=rows.Scan(&tag.Id,&tag.Title,&tag.Language,&tag.Release_date,&tag.Poster_path,&tag.Background_path,&tag.Overview,&tag.Genre_ids)
if err != nil {
fmt.Println(err)
}
fmt.Println(tag.Id)
list.Results = append(list.Results,tag)
}
err = rows.Err()
if err != nil {
fmt.Println(err)
}
defer db.Close()
//fmt.Fprintf(w, "Hello, %q\n", list.Results[3])
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(list)
//spew.Dump(list)
//fmt.Fprintf(w, "given lamguage, %q\n", tag.Poster_path)
}
func main() {
handleRequests()
}
http.FileServer() should not be called in a function like that. It returns a Handler function (a function similar to the posterfunc you created).
It should be used as the handler function in the route configuration like this:
myRouter.HandleFunc("/poster_path/",http.FileServer(http.Dir("./your_dir")))
Here you can find the documentation for http.FileServer, with some more detailed examples.
This blog post does a fine step-by-step explanation on how to set it up.
UPDATE:
If you want to use it inside your handler, you should use http.ServeFile. It will respond to the request with the contents of the named file or directory.
Docs here
func posterfunc(w http.ResponseWriter, r *http.Request){
w.Header().Set("Content-Type", "image/jpeg")
vars := mux.Vars(r)
key := vars["id"]
var url = "/home/rakshithjk/Desktop/poster/"+key+".jpg"
http.ServeFile(w, r, url)
}
UPDATE 2:
The issue in the new snippet is that you are printing to the interface just before serving the file.
Remove this line:
fmt.Fprintf(w, "Hello, %q\n", url)

Good way to disable directory listing with http.FileServer in Go

If you use the http.FileServer in Go like:
func main() {
port := flag.String("p", "8100", "port to serve on")
directory := flag.String("d", ".", "the directory of static file to host")
flag.Parse()
http.Handle("/", http.FileServer(http.Dir(*directory)))
log.Printf("Serving %s on HTTP port: %s\n", *directory, *port)
log.Fatal(http.ListenAndServe(":"+*port, nil))
}
Then accessing a directory will give you a listing of files. Often this is disabled for web services and instead responds with 404 and I would like this behaviour too.
http.FileServer has no options for this AFAIK and I have seen a proposed way to solve this here https://groups.google.com/forum/#!topic/golang-nuts/bStLPdIVM6w what they do is wrapping the http.FileSystem type and implementing an own Open method. However this doesn't give a 404 when the path is a directory, it just gives a blank page, and it's unclear how to modify it to accomodate this. This is what they do:
type justFilesFilesystem struct {
fs http.FileSystem
}
func (fs justFilesFilesystem) Open(name string) (http.File, error) {
f, err := fs.fs.Open(name)
if err != nil {
return nil, err
}
return neuteredReaddirFile{f}, nil
}
type neuteredReaddirFile struct {
http.File
}
func (f neuteredReaddirFile) Readdir(count int) ([]os.FileInfo, error) {
return nil, nil
}
func main() {
fs := justFilesFilesystem{http.Dir("/tmp/")}
http.ListenAndServe(":8080", http.FileServer(fs))
}
Note: if you make Readdir return nil, os.ErrNotExist then you get a 500 response with "Error reading directory" - not 404.
Any ideas on how to neatly present a 404 and still preserving the feature of automatically finding an index.html if present?
This behavior can be changed if you substitute not a Readdir method, but the Stat.
Please take a look at working code below. It supports serving of index.html files if they are inside of requested directory and returns 404 in case there is no index.html and it is a directory.
package main
import (
"io"
"net/http"
"os"
)
type justFilesFilesystem struct {
fs http.FileSystem
// readDirBatchSize - configuration parameter for `Readdir` func
readDirBatchSize int
}
func (fs justFilesFilesystem) Open(name string) (http.File, error) {
f, err := fs.fs.Open(name)
if err != nil {
return nil, err
}
return neuteredStatFile{File: f, readDirBatchSize: fs.readDirBatchSize}, nil
}
type neuteredStatFile struct {
http.File
readDirBatchSize int
}
func (e neuteredStatFile) Stat() (os.FileInfo, error) {
s, err := e.File.Stat()
if err != nil {
return nil, err
}
if s.IsDir() {
LOOP:
for {
fl, err := e.File.Readdir(e.readDirBatchSize)
switch err {
case io.EOF:
break LOOP
case nil:
for _, f := range fl {
if f.Name() == "index.html" {
return s, err
}
}
default:
return nil, err
}
}
return nil, os.ErrNotExist
}
return s, err
}
func main() {
fs := justFilesFilesystem{fs: http.Dir("/tmp/"), readDirBatchSize: 2}
fss := http.FileServer(fs)
http.ListenAndServe(":8080", fss)
}

Go race condition in timeout handler

I can see two main issues in the example code below, but I don't know how to solve them correctly.
If the timeout handler does not get the signal through the errCh that the next handler has completed or an error occured, it will reply "408 Request timeout" to the request.
The problem here is that the ResponseWriter is not safe to be used by multiple goroutines. And the timeout handler starts a new goroutine when executing the next handler.
Issues:
How to prevent the next handler from writing into the ResponseWriter when the ctx's Done channel times out in the timeout handler.
How to prevent the timeout handler from replying 408 status code when the next handler is writing into the ResponseWriter but it has not finished yet and the ctx's Done channel times out in the timeout handler.
package main
import (
"context"
"fmt"
"net/http"
"time"
)
func main() {
http.Handle("/race", handlerFunc(timeoutHandler))
http.ListenAndServe(":8080", nil)
}
func timeoutHandler(w http.ResponseWriter, r *http.Request) error {
const seconds = 1
ctx, cancel := context.WithTimeout(r.Context(), time.Duration(seconds)*time.Second)
defer cancel()
r = r.WithContext(ctx)
errCh := make(chan error, 1)
go func() {
// w is not safe for concurrent use by multiple goroutines
errCh <- nextHandler(w, r)
}()
select {
case err := <-errCh:
return err
case <-ctx.Done():
// w is not safe for concurrent use by multiple goroutines
http.Error(w, "Request timeout", 408)
return nil
}
}
func nextHandler(w http.ResponseWriter, r *http.Request) error {
// just for fun to simulate a better race condition
const seconds = 1
time.Sleep(time.Duration(seconds) * time.Second)
fmt.Fprint(w, "nextHandler")
return nil
}
type handlerFunc func(w http.ResponseWriter, r *http.Request) error
func (fn handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if err := fn(w, r); err != nil {
http.Error(w, "Server error", 500)
}
}
Here is a possible solution, which is based on #Andy's comment.
A new responseRecorder will be passed to the nextHandler, and the recorded response will be copied back to the client:
func timeoutHandler(w http.ResponseWriter, r *http.Request) error {
const seconds = 1
ctx, cancel := context.WithTimeout(r.Context(),
time.Duration(seconds)*time.Second)
defer cancel()
r = r.WithContext(ctx)
errCh := make(chan error, 1)
w2 := newResponseRecorder()
go func() {
errCh <- nextHandler(w2, r)
}()
select {
case err := <-errCh:
if err != nil {
return err
}
w2.cloneHeader(w.Header())
w.WriteHeader(w2.status)
w.Write(w2.buf.Bytes())
return nil
case <-ctx.Done():
http.Error(w, "Request timeout", 408)
return nil
}
}
And here is the responseRecorder:
type responseRecorder struct {
http.ResponseWriter
header http.Header
buf *bytes.Buffer
status int
}
func newResponseRecorder() *responseRecorder {
return &responseRecorder{
header: http.Header{},
buf: &bytes.Buffer{},
}
}
func (w *responseRecorder) Header() http.Header {
return w.header
}
func (w *responseRecorder) cloneHeader(dst http.Header) {
for k, v := range w.header {
tmp := make([]string, len(v))
copy(tmp, v)
dst[k] = tmp
}
}
func (w *responseRecorder) Write(data []byte) (int, error) {
if w.status == 0 {
w.WriteHeader(http.StatusOK)
}
return w.buf.Write(data)
}
func (w *responseRecorder) WriteHeader(status int) {
w.status = status
}

implement tls.Config.GetCertificate with self signed certificates

I m trying to figure out how i can implement a function to feed to tls.Config.GetCertificate with self signed certificates.
I used this bin source as a base, https://golang.org/src/crypto/tls/generate_cert.go
Also read this,
https://ericchiang.github.io/tls/go/https/2015/06/21/go-tls.html
Unfortunately, so far i m stuck with this error
2016/11/03 23:18:20 http2: server: error reading preface from client 127.0.0.1:34346: remote error: tls: unknown certificate authority
I think i need to generate a CA cert and then sign the key with it, but i m not sure how to proceed (....).
Here is my code, can someone help with that ?
package gssc
import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"github.com/pkg/errors"
"math/big"
"net"
"strings"
"time"
)
func GetCertificate(arg interface{}) func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
var opts Certopts
var err error
if host, ok := arg.(string); ok {
opts = Certopts{
RsaBits: 2048,
Host: host,
ValidFrom: time.Now(),
}
} else if o, ok := arg.(Certopts); ok {
opts = o
} else {
err = errors.New("Invalid arg type, must be string(hostname) or Certopt{...}")
}
return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if err != nil {
return nil, err
}
return generate(opts)
}
}
type Certopts struct {
RsaBits int
Host string
IsCA bool
ValidFrom time.Time
ValidFor time.Duration
}
func generate(opts Certopts) (*tls.Certificate, error) {
priv, err := rsa.GenerateKey(rand.Reader, opts.RsaBits)
if err != nil {
return nil, errors.Wrap(err, "failed to generate private key")
}
notAfter := opts.ValidFrom.Add(opts.ValidFor)
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return nil, errors.Wrap(err, "Failed to generate serial number\n")
}
template := x509.Certificate{
SerialNumber: serialNumber,
Subject: pkix.Name{
Organization: []string{"Acme Co"},
},
NotBefore: opts.ValidFrom,
NotAfter: notAfter,
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
hosts := strings.Split(opts.Host, ",")
for _, h := range hosts {
if ip := net.ParseIP(h); ip != nil {
template.IPAddresses = append(template.IPAddresses, ip)
} else {
template.DNSNames = append(template.DNSNames, h)
}
}
if opts.IsCA {
template.IsCA = true
template.KeyUsage |= x509.KeyUsageCertSign
}
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return nil, errors.Wrap(err, "Failed to create certificate")
}
return &tls.Certificate{
Certificate: [][]byte{derBytes},
PrivateKey: priv,
}, nil
}
This is the test code i use
package main
import (
"crypto/tls"
"github.com/mh-cbon/gssc"
"net/http"
)
type ww struct{}
func (s *ww) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("This is an example server.\n"))
}
func main() {
s := &http.Server{
Handler: &ww{},
Addr: ":8080",
TLSConfig: &tls.Config{
InsecureSkipVerify: true,
GetCertificate: gssc.GetCertificate("example.org"),
},
}
s.ListenAndServeTLS("", "")
}
Thanks a lot!
Your implementation of tls.Config.GetCertificate is causing the problem.
You are generating a certificate each time tls.Config.GetCertificate is called. You need to generate the certificate once and then return it in the anonymous function.
In gssc.GetCertificate :
cert, err := generate(opts)
return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if err != nil {
return nil, err
}
return cert, err
}

The method of reading an integral network PDU in Go

I am developing a simple Go server program which receives client's request and process it. And the code is simplified as this:
package main
import (
"fmt"
"net"
"os"
)
const (
pduLen = 32
)
func checkError(err error) {
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func main() {
var buffer [4096]byte
var count int
conn, err := net.Dial("tcp", fmt.Sprintf("%s:%s", os.Args[1], os.Args[2]))
checkError(err)
for count < pduLen {
n, err := conn.Read(buffer[count:])
checkError(err)
count += n
}
......
}
I assume every request's length is 32 bytes (just an example). Because the TCP is a stream protocol, I need to use a loop to check whether an integral PDU is read:
for count < pduLen {
n, err := conn.Read(buffer[count:])
checkError(err)
count += n
}
Is there any method to assure that an integral PDU is read? Personally, I think the loop code is a little ugly.
It can depend on the exact nature of the PDU you are receiving, but this example will look for the size, and then read everything (using io.ReadFul()).
func read(conn net.Conn, key string) string {
fmt.Fprintf(conn, GenerateCommand(OP_GET, key))
if verify(conn) {
var size uint16
binary.Read(conn, binary.LittleEndian, &size)
b := make([]byte, size)
// _, err := conn.Read(b)
_, err := io.ReadFull(conn, b)
if err == nil {
return string(b)
}
}
return ""
}
func verify(conn net.Conn) bool {
b := make([]byte, 1)
conn.Read(b)
return b[0] == ERR_NO_ERROR
}
Used in:
conn, err := net.Dial("tcp", ":12345")
if err != nil {
t.Error(err)
}
write(conn, "foo", "bar")
if !verify(conn) {
t.Error("Bad write!")
}
if r := read(conn, "foo"); r != "bar" {
t.Errorf("Bad read! Got %v", r)
}
After discussing this issue in golang-nuts: How to read an integral network PDU?
The code should be:
import "io"
......
pdu := make([]byte, pduLen)
io.ReadFull(conn, pdu)

Resources