Go httpClient memory leaks - http

UPDATED CODE
Hi I have memory leaks in httpClient, i've added sync.WaitGroup and now I see that goroutine with httpClient don't closed. How to fix it?
func checkProxySOCKS(prox string, c chan QR, wg *sync.WaitGroup) (err error) {
defer wg.Done()
dialer, _ := proxy.SOCKS5("tcp", prox, nil, proxy.Direct)
timeout := time.Duration(2 * time.Second)
httpClient := &http.Client{
Timeout: timeout,
Transport: &http.Transport{
DisableKeepAlives: true,
Dial: dialer.Dial,
},
}
res, err := httpClient.Get("https://telegram.org/")
if err != nil {
c <- QR{Addr: prox, Res: false}
return
}
defer res.Body.Close()
io.Copy(ioutil.Discard, res.Body)
c <- QR{Addr: prox, Res: true}
return nil
}
Here i create goroutines
for _, proxy := range splitedProxies {
wg.Add(1)
go checkProxySOCKS(proxy, respChan, &wg)
}
for range splitedProxies {
wg.Add(1)
r := <-respChan
if r.Res {
checkedProxiesArray = append(checkedProxiesArray, r.Addr)
}
wg.Done()
}
wg.Wait()

I'm read comments, so this should solve your problem
const (
timeout = time.Duration(1000 * time.Millisecond)
tt = time.Duration(100 * time.Millisecond)
)
func checkProxySOCKS(prox string, c chan QR, wg *sync.WaitGroup) (err error) {
defer wg.Done()
d := net.Dialer{
Timeout: tt,
KeepAlive: tt,
}
dialer, _ := proxy.SOCKS5("tcp", prox, nil, &d)
httpClient := &http.Client{
Timeout: timeout,
Transport: &http.Transport{
DisableKeepAlives: true,
Dial: dialer.Dial,
},
}
res, err := httpClient.Get("https://telegram.org/")
if err != nil {
c <- QR{Addr: prox, Res: false}
return
}
defer res.Body.Close()
io.Copy(ioutil.Discard, res.Body)
c <- QR{Addr: prox, Res: true}
return nil
}

Related

SQL query hangs or crashes

Following code on Go 1.16.6 hangs on last Exec call (or crashes if same functions are called from different goroutines)
Both libraries "github.com/mattn/go-sqlite3" and "modernc.org/sqlite" give same results
package main
import (
"os"
"testing"
"database/sql"
_ "github.com/mattn/go-sqlite3"
// _ "modernc.org/sqlite"
)
func Test_Bug3(t *testing.T) {
DBPath := "test.db"
os.Remove(DBPath)
DB, err := sql.Open("sqlite3", DBPath)
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
if _, err := DB.Exec(`CREATE TABLE IF NOT EXISTS verdictcache (sha1 text);`); err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
_, err = DB.Exec("INSERT OR REPLACE INTO verdictcache (sha1) VALUES ($1)", "a")
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
_, err = DB.Query("SELECT * FROM verdictcache WHERE sha1=$1", "a")
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
_, err = DB.Exec("INSERT OR REPLACE INTO verdictcache (sha1) VALUES ($1)", "b")
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
}
What is happening is almost certainly that you did not call Close() (or otherwise consumed the rows) on the result returned by DB.Query(...).
Try:
func Test_Bug3(t *testing.T) {
DBPath := "test.db"
os.Remove(DBPath)
DB, err := sql.Open("sqlite3", DBPath)
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
if _, err := DB.Exec(`CREATE TABLE IF NOT EXISTS verdictcache (sha1 text);`); err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
_, err = DB.Exec("INSERT OR REPLACE INTO verdictcache (sha1) VALUES ($1)", "a")
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
res, err := DB.Query("SELECT * FROM verdictcache WHERE sha1=$1", "a")
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
res.Close()
_, err = DB.Exec("INSERT OR REPLACE INTO verdictcache (sha1) VALUES ($1)", "b")
if err != nil {
t.Fatalf("%s: %v", DBPath, err)
}
}

How to open tab in new window of the same browser with chromedp?

How to open tab in new window of the same browser?
Some web applications do not work in an inactive tab.
In the example, several windows open, but only the first window is available for management. When creating the rest, an error occurs
chrome failed to start: A window or tab will open in the current browser session.
package main
import (
"context"
"log"
"time"
"github.com/chromedp/chromedp"
)
func main() {
userDir := "someUserDir"
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.DisableGPU,
chromedp.UserDataDir(userDir),
chromedp.WindowSize(1368, 768),
chromedp.Flag("headless", false),
chromedp.Flag("enable-automation", false),
chromedp.Flag("restore-on-startup", false),
chromedp.Flag("new-window", true),
)
for i := 0; i < 5; i++ {
log.Printf("open window %d", i)
allocCtx, _ := chromedp.NewExecAllocator(context.Background(), opts...)
ctx, _ := chromedp.NewContext(allocCtx)
if err := chromedp.Run(ctx, chromedp.Navigate("https://example.com")); err != nil {
log.Println(err)
}
}
time.Sleep(time.Minute)
}
package main
import (
"context"
"log"
"time"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/chromedp"
)
func main() {
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.DisableGPU,
chromedp.UserDataDir("someUserDir"),
chromedp.Flag("headless", false),
chromedp.Flag("enable-automation", false),
chromedp.Flag("restore-on-startup", false),
)
allocCtx, _ := chromedp.NewExecAllocator(context.Background(), opts...)
ctx, _ := chromedp.NewContext(allocCtx)
if err := chromedp.Run(ctx, chromedp.Navigate("about:blank")); err != nil {
log.Fatalln(err)
}
for i := 0; i < 5; i++ {
var res *runtime.RemoteObject
if err := chromedp.Run(ctx, chromedp.Evaluate(`window.open("about:blank", "", "resizable,scrollbars,status")`, &res)); err != nil {
log.Fatalln(err)
}
targets, err := chromedp.Targets(ctx)
if err != nil {
log.Fatalln(err)
}
for _, t := range targets {
if !t.Attached {
newCtx, _ := chromedp.NewContext(ctx, chromedp.WithTargetID(t.TargetID))
if err := chromedp.Run(newCtx, chromedp.Navigate("https://example.com")); err != nil {
log.Fatalln(err)
}
}
}
}
time.Sleep(time.Minute)
}

Calling winapi function in golang with struct union

I am trying to call the TransmitPackets function on windows using GO.
The goal is to be able to send multiple packets with one syscall (can't be achieved with WSASend [it'll send fragmented IP packets]).
My code panics
panic: write udp 192.168.1.26:51817->8.8.8.8:8000: transmitpackets: An invalid argument was supplied.
goroutine 1 [running]:
main.main()
c:/Users/amit/dev/go/src/rio/main.go:26 +0x210
exit status 2
Process exiting with code: 1
Here's my test code
package main
import (
"math/rand"
"net"
)
func main() {
raddr, err := net.ResolveUDPAddr("udp", "8.8.8.8:8000")
if err != nil {
panic(err)
}
con, err := net.DialUDP("udp", nil, raddr)
if err != nil {
panic(err)
}
packets := make(net.Buffers, 10)
for i := 0; i < len(packets); i++ {
packets[i] = make([]byte, 1400)
rand.Read(packets[i])
}
_, err = con.WriteMultiple(packets)
if err != nil {
panic(err)
}
}
and Here's my call to TransmitPackets:
type TransmitPacketsElement struct {
dwElFlags uint32
cLength uint32
pBuffer unsafe.Pointer
nFileOffset uint64
hFile uintptr
}
func transmitPackets(s Handle, bufs [][]byte, overlapped *Overlapped) (err error) {
var maxPacketLen = 0
tpElements := make([]TransmitPacketsElement, len(bufs))
for i, tpElement := range tpElements {
buffer := bufs[i]
if len(buffer) > maxPacketLen {
maxPacketLen = len(buffer)
}
tpElement.cLength = uint32(len(buffer))
tpElement.dwElFlags = uint32(uint32(TP_ELEMENT_MEMORY) | uint32(TP_ELEMENT_EOP))
tpElement.pBuffer = unsafe.Pointer(&buffer[0])
}
r1, _, e1 := Syscall6(transmitPacketsFunc.addr, 6, uintptr(s), uintptr(unsafe.Pointer(&tpElements[0])), uintptr(uint32(len(tpElements))), uintptr(uint32(maxPacketLen)), uintptr(unsafe.Pointer(overlapped)), 0)
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = EINVAL
}
}
return
}
You can see the full implementation in my modified go 1.8.3 source on github

Porting Python AES encryption routies to Golang

I am trying to port the following Python AES file encryption routines over to Go:
def derive_key_and_iv(password, salt, key_length, iv_length):
d = d_i = ''
while len(d) < key_length + iv_length:
d_i = md5(d_i + password + salt).digest()
d += d_i
return d[:key_length], d[key_length:key_length+iv_length]
def encrypt(in_file, out_file, password, key_length=32):
bs = AES.block_size
salt = Random.new().read(bs - len('Salted__'))
key, iv = derive_key_and_iv(password, salt, key_length, bs)
cipher = AES.new(key, AES.MODE_CBC, iv)
out_file.write('Salted__' + salt)
finished = False
while not finished:
chunk = in_file.read(1024 * bs)
if len(chunk) == 0 or len(chunk) % bs != 0:
padding_length = (bs - len(chunk) % bs) or bs
chunk += padding_length * chr(padding_length)
finished = True
out_file.write(cipher.encrypt(chunk))
def decrypt(in_file, out_file, password, key_length=32):
bs = AES.block_size
print(bs)
salt = in_file.read(bs)[len('Salted__'):]
key, iv = derive_key_and_iv(password, salt, key_length, bs)
cipher = AES.new(key, AES.MODE_CBC, iv)
next_chunk = ''
finished = False
while not finished:
chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(1024 * bs))
if len(next_chunk) == 0:
padding_length = ord(chunk[-1])
chunk = chunk[:-padding_length]
finished = True
out_file.write(chunk)
I have the following Go routines coded up but I'm not quite able to get it working. I am trying to get the encryption routines working in Go for callers that call the decrypt in Python and C so I'm really only interested in figuring out how to get my Golang encryption routine working but have included all the Python bits for clarity.
My current Go routines look like this:
package main
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"io"
"log"
"crypto/md5"
"os"
)
func pyEncrypt(password []byte, pathToInFile string, pathToOutFile string){
bs := int(aes.BlockSize)
salt := make([]byte, aes.BlockSize - len("Salted__"))
_, err := rand.Read(salt)
if err != nil {panic(err)}
key, iv := deriveKeyAndIV(password, salt, bs)
block, err := aes.NewCipher(key)
if err != nil {panic(err)}
cfb := cipher.NewCFBEncrypter(block, iv)
fin, err := os.Open(pathToInFile)
if err != nil {panic(err)}
defer fin.Close()
fout, err := os.Create(pathToOutFile)
if err != nil {panic(err)}
defer fout.Close()
_, err = fout.Write([]byte("Salted__"))
if err != nil {panic(err)}
_, err = fout.Write(salt)
if err != nil {panic(err)}
for true{
ciphertext := make([]byte, 1024 *bs)
chunk := make([]byte, 1024 * bs)
_, err := fin.Read(chunk)
if err == io.EOF{
break
}else if err != nil{
panic(err)
}
cfb.XORKeyStream(ciphertext, chunk)
fout.Write(ciphertext)
}
}
func deriveKeyAndIV(password []byte, salt []byte, bs int) ([]byte, []byte) {
var digest []byte
hash := md5.New()
out := make([]byte, 32 + bs) //right now I'm just matching the default key size (32) from the python script so 32 represents the default from python
for i := 0; i < 32 + bs ; i += len(digest) {
hash.Reset()
hash.Write(digest)
hash.Write(password)
hash.Write(salt)
digest = hash.Sum(digest[:0])
copy(out[i:], digest)
}
return out[:32], out[32:32+bs] //matching the default key size from Python as that is what the application uses
}
func main() {
pwd := []byte("test123")
pyEncrypt(pwd, "/home/chris/pt.txt", "/home/chris/genc.txt")
}
Right now it runs, lol, generates an encrypted file that looks right and the Python "decrypts" without error but it generates gibberish and doesn't actually produce the clear text. The Python routines work by stand-alone but I can't get my Golang encrypt producing output that the Python decrypt can decrypt. I have to match the Python encryption routine for compatibility with the callers. Do you see what I'm doing wrong in my Golang encryption routine? Thank you so much for your assistance.
Here is the working encryption routine:
func pyEncrypt(password string, pathToInFile string, pathToOutFile string){
bs := int(aes.BlockSize)
salt := make([]byte, aes.BlockSize - len("Salted__"))
_, err := rand.Read(salt)
if err != nil {panic(err)}
key, iv := deriveKeyAndIV([]byte(password), salt, bs)
block, err := aes.NewCipher(key)
if err != nil {panic(err)}
cbc := cipher.NewCBCEncrypter(block, iv)
fin, err := os.Open(pathToInFile)
if err != nil {panic(err)}
defer fin.Close()
fout, err := os.Create(pathToOutFile)
if err != nil {panic(err)}
defer fout.Close()
_,err = fout.Write([]byte("Salted__"))
if err != nil {panic(err)}
_,err = fout.Write(salt)
if err != nil {panic(err)}
for true{
chunk := make([]byte, 1024 * bs)
n,err := fin.Read(chunk)
if err == io.EOF{
break
}else if err != nil{
panic(err)
}
if n == 0 || n % bs != 0 {//need to pad up to block size :bs
paddingLength := (bs - n % bs)
paddingChr := []byte(string(rune(paddingLength)))
paddingBytes := make([]byte, paddingLength)
for i := 0; i < paddingLength; i++{
paddingBytes[i] = paddingChr[0]
}
chunk = append(chunk[0:n], []byte(paddingBytes)...)
}else{
chunk = chunk[0:n]//no padding needed
}
ciphertext := make([]byte, len(chunk))
cbc.CryptBlocks(ciphertext,chunk)
fout.Write(ciphertext)
}
}

Asynchronous messages golang

I have a golang server doing something like this:
package main
func main() {
for {
c := listener.Accept()
go handle(c)
}
}
...
func handle(c net.Conn) {
m := readMessage(c) // func(net.Conn)Message
r := processMessage(m) //func(Message)Result
sendResult(c, r) // func(net.Conn,Result)
}
Which reads and writes messages synchronously. What I need now is to send messages asynchronously through a given open connection, I know a channel can be used by I'm kind of lost.
This is my idea:
...
func someWhereElese(c chan Result) {
// generate a message and a result
r := createResultFromSomewhere()
c <- r // send the result through the channel
}
And modify my handle to use that same channel instead
func handle(c net.Conn, rc chan Result) {
m := readMessage(c) // func(net.Conn)Message
r := processMessage(m) //func(Message)Result
//sendResult(c, r) // func(net.Conn,Result)
rc <- r
}
And here's where my confusion lies.
The result channel should be created and it should have a connection where to send whatever it receives
func doSend(c net.Con, rc chan Result) {
r := rc // got a result channel
sendResult(c, r) // send it through the wire
}
But where should that channel be created? In the main loop?
func main() {
...
for {
c := l.Accept()
rc := make(chan Result)
go doSend(c, rc)
}
}
What about the read? Should it go in it's own channel/gorutine?
If I need to broadcast to n clients, should I keep a slice of result channels? a slice of connections?
I'm kind of confused here, but I feel I'm close.
This program seems to solve my immediate question
package main
import (
"bytes"
"encoding/binary"
"log"
"net"
)
var rcs []chan int = make([]chan int,0)
func main() {
a, e := net.ResolveTCPAddr("tcp", ":8082")
if e != nil {
log.Fatal(e)
}
l, e := net.ListenTCP("tcp", a)
for {
c, e := l.Accept()
if e != nil {
log.Fatal(e)
}
rc := make(chan int)
go read(c, rc)
go write(c, rc)
rcs = append(rcs, rc)
// simulate broacast
log.Println(len(rcs))
if len(rcs) > 5 {
func() {
for _, v := range rcs {
log.Println("sending")
select {
case v <- 34:
log.Println("done sending")
default:
log.Println("didn't send")
}
}
}()
}
}
}
func read(c net.Conn, rc chan int) {
h := make([]byte, 2)
for {
_, err := c.Read(h)
if err != nil {
rc <- -1
}
var v int16
binary.Read(bytes.NewReader(h[:2]), binary.BigEndian, &v)
rc <- int(v)
}
}
func write(c net.Conn, rc chan int) {
for {
r := <-rc
o := []byte{byte(r * 2)}
c.Write(o)
}
}

Resources