TCP client / server file transfer in Go - tcp

I'm new to Go and I'm having troubles debugging this client / server file transfer code. When I request a 719kb png file from the server, I get the 719kb.. but not perfectly, the png when I open it isn't completely displayed (some is cut off. Where am I going wrong here?
// CLIENT ///
package main
import (
"bufio"
"bytes"
"fmt"
"io"
"log"
"net"
"os"
"strings"
)
const BUFFER_SIZE = 1024
func main() {
//get port and ip address to dial
if len(os.Args) != 3 {
fmt.Println("useage example: tcpClient 127.0.0.1 7005")
return
}
var ip string = os.Args[1]
var port string = os.Args[2]
connection, err := net.Dial("tcp", ip+":"+port)
if err != nil {
fmt.Println("There was an error making a connection")
}
//read from
reader := bufio.NewReader(os.Stdin)
fmt.Print("Please enter 'get <filename>' or 'send <filename>' to transfer files to the server\n\n")
inputFromUser, _ := reader.ReadString('\n')
arrayOfCommands := strings.Split(inputFromUser, " ")
if arrayOfCommands[0] == "get" {
getFileFromServer(arrayOfCommands[1], connection)
} else if arrayOfCommands[0] == "send" {
sendFileToServer(arrayOfCommands[1], connection)
} else {
fmt.Println("Bad Command")
}
}
func sendFileToServer(fileName string, connection net.Conn) {
var currentByte int64 = 0
fmt.Println("send to client")
fileBuffer := make([]byte, BUFFER_SIZE)
var err error
//file to read
file, err := os.Open(strings.TrimSpace(fileName)) // For read access.
if err != nil {
connection.Write([]byte("-1"))
log.Fatal(err)
}
connection.Write([]byte("send " + fileName))
//read file until there is an error
for err == nil || err != io.EOF {
_, err = file.ReadAt(fileBuffer, currentByte)
currentByte += BUFFER_SIZE
fmt.Println(fileBuffer)
connection.Write(fileBuffer)
}
file.Close()
connection.Close()
}
func getFileFromServer(fileName string, connection net.Conn) {
var currentByte int64 = 0
fileBuffer := make([]byte, BUFFER_SIZE)
var err error
file, err := os.Create(strings.TrimSpace(fileName))
if err != nil {
log.Fatal(err)
}
connection.Write([]byte("get " + fileName))
for {
connection.Read(fileBuffer)
cleanedFileBuffer := bytes.Trim(fileBuffer, "\x00")
_, err = file.WriteAt(cleanedFileBuffer, currentByte)
currentByte += BUFFER_SIZE
if err == io.EOF {
break
}
}
file.Close()
return
}
// END CLIENT //
// SERVER //
package main
import (
"bytes"
"fmt"
"io"
"log"
"net"
"os"
"strings"
)
const BUFFER_SIZE = 1024
const PORT = "7005"
func main() {
fmt.Println("start listening")
server, error := net.Listen("tcp", "localhost:"+PORT)
if error != nil {
fmt.Println("There was an error starting the server" + error.Error())
return
}
//infinate loop
for {
connection, error := server.Accept()
if error != nil {
fmt.Println("There was am error with the connection" + error.Error())
return
}
fmt.Println("connected")
//handle the connection, on it's own thread, per connection
go connectionHandler(connection)
}
}
func connectionHandler(connection net.Conn) {
buffer := make([]byte, BUFFER_SIZE)
_, error := connection.Read(buffer)
if error != nil {
fmt.Println("There is an error reading from connection", error.Error())
return
}
fmt.Println("command recieved: " + string(buffer))
//loop until disconntect
cleanedBuffer := bytes.Trim(buffer, "\x00")
cleanedInputCommandString := strings.TrimSpace(string(cleanedBuffer))
arrayOfCommands := strings.Split(cleanedInputCommandString, " ")
fmt.Println(arrayOfCommands[0])
if arrayOfCommands[0] == "get" {
sendFileToClient(arrayOfCommands[1], connection)
} else if arrayOfCommands[0] == "send" {
fmt.Println("getting a file")
getFileFromClient(arrayOfCommands[1], connection)
} else {
_, error = connection.Write([]byte("bad command"))
}
}
func sendFileToClient(fileName string, connection net.Conn) {
var currentByte int64 = 0
fmt.Println("send to client")
fileBuffer := make([]byte, BUFFER_SIZE)
//file to read
file, err := os.Open(strings.TrimSpace(fileName)) // For read access.
if err != nil {
log.Fatal(err)
}
var err2 error
//read file until there is an error
for {
_, err2 = file.ReadAt(fileBuffer, currentByte)
currentByte += BUFFER_SIZE
fmt.Println(fileBuffer)
connection.Write(fileBuffer)
if err2 == io.EOF {
break
}
}
file.Close()
return
}
func getFileFromClient(fileName string, connection net.Conn) {
var currentByte int64 = 0
fileBuffer := make([]byte, BUFFER_SIZE)
var err error
file, err := os.Create(strings.TrimSpace(fileName))
if err != nil {
log.Fatal(err)
}
connection.Write([]byte("get " + fileName))
for err == nil || err != io.EOF {
connection.Read(fileBuffer)
cleanedFileBuffer := bytes.Trim(fileBuffer, "\x00")
_, err = file.WriteAt(cleanedFileBuffer, currentByte)
if len(string(fileBuffer)) != len(string(cleanedFileBuffer)) {
break
}
currentByte += BUFFER_SIZE
}
connection.Close()
file.Close()
return
}
// END SERVER //

You need to account for the number of bytes returned from ReadAt, otherwise the last fileBuffer you send will have extra garbage bytes.
Example:
n, err := file.ReadAt(fileBuffer, currentByte)
connection.Write(fileBuffer[:n])
Also bytes.Trim(fileBuffer, "\x00") will destroy almost any binary file since usually they use null bytes to fill space.
Also the proper way of doing this is just using io.Copy:
file, err := os.Open(strings.TrimSpace(fileName)) // For read access.
if err != nil {
log.Fatal(err)
}
defer file.Close() // make sure to close the file even if we panic.
n, err = io.Copy(connection, file)
if err != nil {
log.Fatal(err)
}
fmt.Println(n, "bytes sent")

Related

How to check whether a URL is downloadable or not in golang?

I'm trying to download a file fro the url to a local file.
I wanted to test whether the requesting url is only file, if it is not a file it should return bad request
Any help could be appreciated
package main
import (
"fmt"
"io"
"net/http"
"os"
)
func main() {
fileUrl := "http://example.com/file.txt"
err := DownloadFile("./example.txt", fileUrl)
if err != nil {
panic(err)
}
fmt.Println("Downloaded: " + fileUrl)
}
// DownloadFile will download a url to a local file.
func DownloadFile(filepath string, url string) error {
// Get the data
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
// Create the file
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
return err
}
Below is the way to check whether the URL is downloadable or not. Hope this helps someone :)
package main
import (
"fmt"
"io"
"net/http"
"os"
)
func main() {
fileUrl := "http://example.com/file.txt"
err := DownloadFile("./example.txt", fileUrl)
if err != nil {
panic(err)
}
fmt.Println("Downloaded: " + fileUrl)
}
// DownloadFile will download a url to a local file.
func DownloadFile(filepath string, url string) error {
// Get the data
resp, err := http.Get(url)
contentType = resp.Header.Get("Content-Type")
if err != nil {
return err
}
defer resp.Body.Close()
if contentType == "application/octet-stream" {
// Create the file
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
return err
} else {
fmt.Println("Requested URL is not downloadable")
return nil
}
}

Stream HAR events

I have a long running app that I'd like to monitor in real time. HAR files allow me to do this after the fact, but as they are an "archive", they don't allow me to do this in real time.
Is their anyway to stream the "events" array of the HAR file so I can process them as they are generated?
This can be firefox or chrome.
So with some help from https://github.com/mafredri/cdp/tree/master/example/screencast I figured out how to do this in go with chrome's debugger api
What this code doesn't do is tie the request body to the response (where it isn't available), but as I show the RequestID will be consistent so if one serializes event processing (say via locking) one can save the body and use it when the response event is seen.
package main
import (
"context"
"log"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/cdpcmd"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/rpcc"
)
func main() {
if err := run(); err != nil {
panic(err)
}
}
func run() error {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
devt := devtool.New("http://localhost:9222")
page, err := devt.Get(ctx, devtool.Page)
if err != nil {
return err
}
conn, err := rpcc.DialContext(ctx, page.WebSocketDebuggerURL)
if err != nil {
return err
}
defer conn.Close()
c := cdp.NewClient(conn)
err = c.Page.Enable(ctx)
if err != nil {
return err
}
loadEventFired, err := c.Page.LoadEventFired(ctx)
if err != nil {
return err
}
_, err = c.Page.Navigate(ctx, cdpcmd.NewPageNavigateArgs("https://github.com/"))
if err != nil {
return err
}
_, err = loadEventFired.Recv()
if err != nil {
return err
}
loadEventFired.Close()
a := &cdpcmd.NetworkEnableArgs{}
a.SetMaxResourceBufferSize(32000)
a.SetMaxTotalBufferSize(96000)
err = c.Network.Enable(ctx, a)
responseEvents, err := c.Network.ResponseReceived(ctx)
requestEvents, err := c.Network.RequestWillBeSent(ctx)
go func() {
defer responseEvents.Close()
for {
ev, err := responseEvents.Recv()
if err != nil {
log.Printf("Failed to receive network event: %v", err)
return
}
log.Printf("requestid = %v, url = %v", ev.RequestID, ev.Response.URL)
}
}()
go func() {
defer requestEvents.Close()
for {
ev, err := requestEvents.Recv()
if err != nil {
log.Printf("Failed to receive network event: %v", err)
return
}
log.Printf("requestid = %v, url = %v", ev.RequestID, ev.Request.URL)
}
}()
select {}
return nil
}

How can you upload files as a []byte in go?

I would like to use golang post request, upload pictures, but I do not want to pass filepath, just want to pass [] byte
The following article are not what I need because they are used os.Open
golang POST data using the Content-Type multipart/form-data
func Upload(url, file string) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
// Add your image file
f, err := os.Open(file)
if err != nil {
return
}
defer f.Close()
fw, err := w.CreateFormFile("image", file)
if err != nil {
return
}
if _, err = io.Copy(fw, f); err != nil {
return
}
// Add the other fields
if fw, err = w.CreateFormField("key"); err != nil {
return
}
if _, err = fw.Write([]byte("KEY")); err != nil {
return
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
Since you use
if _, err = io.Copy(fw, f); err != nil {
return
}
You may as well edit your code to:
Add new import: "bytes"
Change the method signature to func Upload(url string, file []byte) (err error)
Use io.Copy(fw, bytes.NewReader(f))

Reconnect TCP on EOF in Go

I have the following:
//In an init func
if logStashHost != "" {
lsconn, err = net.Dial("tcp", logStashHost)
}
...
ToLogStash(rec, lsconn)
Then Two functions:
func ReadLogStash(conn net.Conn) {
buffer := make([]byte, 256)
for {
_, err := conn.Read(buffer)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(buffer)
}
}
}
func ToLogStash(r *logrow.Record, conn net.Conn) {
b, err := json.Marshal(r)
if err != nil {
fmt.Println(err)
return
}
_, err = fmt.Fprintln(conn, string(b))
if err != nil {
fmt.Println(err)
}
}
Where ReadLogStash is a running goroutine. If the other side closes, I get EOF. What would be a good implementation in ReadLogStash to have it attempt to reestablish the connection every X seconds when it gets an EOF?
Go has channels for synchronization and communication, use them!
Make your connection in a loop, and have it wait for some sort of message to come back on a channel.
...
errCh := make(chan error)
for {
lsconn, err = net.Dial("tcp", logStashHost)
// check error!
go ReadLogStash(lsconn, errCh)
err = <-errCh
if err != nil {
// bad error
break
}
// sleep to backoff on retries?
}
...
func ReadLogStash(conn net.Conn, errCh chan error) {
_, err := io.Copy(os.Stderr, conn)
if err != nil {
fmt.Println(err)
}
// a nil error from io.Copy means you reached EOF.
errCh <- err
}
Unless you have more functionality in ReadLogStash, you can probably just use io.Copy inline, and forget the entire function, but this pattern may come in useful for you anyway.
Here is what I ended up going with, a channel was the right direction:
if logStashHost != "" {
lsc = make(chan *logrow.Record)
go ToLogStash(lsc, logStashHost)
}
...
if lsc != nil {
lsc <- rec
}
...
func ToLogStash(c chan *logrow.Record, logStashHost string) {
var lsconn net.Conn
var enc *json.Encoder
var err error
connect := func() {
for {
lsconn, err = net.Dial("tcp", logStashHost)
if err == nil {
enc = json.NewEncoder(lsconn)
break
}
log.Println(err)
time.Sleep(time.Second)
}
}
connect()
for r := range c {
err = enc.Encode(r)
if err != nil {
lsconn.Close()
log.Println(err)
connect()
}
}
}

How to make a long connection with http.Client?

I try to connect a http server as long connection, like below:
func main() {
request, err := http.NewRequest("GET", "http://long.connection.org:8080/", nil)
request.SetBasicAuth("xxx", "oooo")
http_client := &http.Client{}
response, _ := http_client.Do(request)
var buf []byte
for {
_, err := response.Body.Read(buf)
if err == io.EOF { break }
fmt.Printf("%s", string(buf))
}
}
But read from response.Body always empty. And seems I can't use response.Body to send data to server.
Any one can help?
This seems to work:
package main
import (
"fmt"
"io"
"log"
"net/http"
)
func main() {
request, err := http.NewRequest("GET", "http://www.example.com/", nil)
if err != nil {
log.Fatal(err)
}
http_client := &http.Client{}
response, err := http_client.Do(request)
if err != nil {
log.Fatal(err)
}
buf := make([]byte, 4096) // any non zero value will do, try '1'.
for {
n, err := response.Body.Read(buf)
if n == 0 && err != nil { // simplified
break
}
fmt.Printf("%s", buf[:n]) // no need to convert to string here
}
fmt.Println()
}
Edit: Added forgotten error handling of NewRequest.

Resources