I have a function which should get the page name and print it, for example, if the URL is http://localhost:8080/login.html the function should print login.html
If you only need to parse the URL you can use below:
package main
import (
"fmt"
"net/url"
)
func main() {
URL := "http://localhost:8080/login.html"
name, err := getPageName(URL)
if err != nil {
panic(err)
}
fmt.Println(name)
}
func getPageName(URL string) (string, error) {
u, err := url.Parse(URL)
if err != nil {
return "", err
}
return u.Path[1:], nil // To remove initial /
}
If you need to get page's HTML and parse the title from <head> you can use go-query
package main
import (
"fmt"
"log"
"net/http"
"github.com/PuerkitoBio/goquery"
)
func main() {
URL := "https://stackoverflow.com"
res, err := http.Get(URL)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
log.Fatal(err)
}
title := doc.Find("title").Text()
fmt.Println(title)
}
You can do this to get the page name:
func GetPageName(address string) (string, error) {
u, err := url.Parse(address)
if err != nil {
return "", err
}
params := strings.Split(u.Path, "/")
// Index page
if len(params) == 0 || (len(params) == 1 && params[0] == "") {
return "", nil
} else {
pageName := params[len(params)-1]
// Has trailing slash
if pageName == "" {
return params[len(params)-2], nil
}
// Doesn't have trailing slash
return pageName, nil
}
}
If the url address is the index page address, for example host.com or host.com/ it returns an empty string.
Otherwise returns the page name for the given url. For example test for host.com/test and host.com/test/ and host.com/path/test.
Related
The hope is to quickly parse a very large number of similar URLs (only one 'id' element differs from one to the next) and dump the response body into a channel that will later be queried by the main function and used to build a text file.
Inside the getpageCanal() function, the body seems to be ok, but after that, I don't understand why the channel doesn't properly load the body string.
package main
import (
"fmt"
"io/ioutil"
"net/http"
"os"
)
func main() {
initial := "https://www1.medion.de/downloads/index.pl?op=detail&id="
ending := "&type=treiber&lang=uk"
links := []string{}
os.Remove("dump.txt")
dumpFile, _ := os.Create("dump.txt")
c := make(chan string)
for i := 16000; i < 16004; i++ {
links = append(links, initial+fmt.Sprint(i)+ending)
}
fmt.Println(links[0])
for _, link := range links {
//the hope is to make this a go routine, but first I need to just make it work
getpageCanal(c, link)
}
for el := range c {
fmt.Println(el)
n, err := dumpFile.WriteString(el)
if err != nil {
fmt.Println(err)
}
if n == 0 {
fmt.Println(" nothing written in main")
}
}
}
func getpageCanal(canal chan string, url string) {
defer close(canal)
page, err := http.Get(url)
if err != nil {
fmt.Println("you done fucked up, boy")
}
content, er2 := ioutil.ReadAll(page.Body)
//fmt.Println(content)
if er2 != nil {
fmt.Println(er2)
}
canal <- string(content)
}
After modifying the code as instructed by the first comments (not closing the channel after each call and making the call to the worker function a go routine) I will now provide you with a working version:
package main
import (
"fmt"
"io/ioutil"
"net/http"
"os"
)
func main() {
initial := "https://www1.medion.de/downloads/index.pl?op=detail&id="
ending := "&type=treiber&lang=uk"
links := []string{}
os.Remove("dump.txt")
dumpFile, _ := os.Create("dump.txt")
c := make(chan string)
for i := 16000; i < 16004; i++ {
links = append(links, initial+fmt.Sprint(i)+ending)
}
fmt.Println(links[0])
for _, link := range links {
go getpageCanal(c, link)
}
for el := range c {
fmt.Println(el)
n, err := dumpFile.WriteString(el)
if err != nil {
fmt.Println(err)
}
if n == 0 {
fmt.Println(" nothing written in main")
}
}
}
func getpageCanal(canal chan string, url string) {
//defer close(canal)
page, err := http.Get(url)
if err != nil {
fmt.Println("you done fucked up, boy")
}
content, er2 := ioutil.ReadAll(page.Body)
//fmt.Println(content)
if er2 != nil {
fmt.Println(er2)
}
canal <- string(content)
}
I try to convert a make(map[string]string) into a yaml like that:
Yaml Output desire:
items:
keys1:value1
keys2:value2
keys3:value3
keys4:value4
The keys,values are this listKey map of string. J = string = {"key1":"value1","key2":"value2" }
type Items struct {
items string
ItemsValues map[string][]string
}
func ConvertToYelm(j string){
y := Items{}
var dataJson map[string]string
err := json.Unmarshal([]byte(j), &dataJson)
if err != nil {
fmt.Println(err)
return
}
listKey := make(map[string]string)
for k := range dataJson{
listKey[k] = k
}
yelm, err := yaml.Marshal(listKey)
if err != nil {
fmt.Println(err)
return
}
err = yaml.Unmarshal(yelm, Items)
if err != nil {
fmt.Println(err)
return
}
yeml2, err := yaml.Marshal(&yelm)
fmt.Printf ("%s", string(yeml2))
To be honest, I'm a little bit lost here, thank you for the help
To get the exact YAML from your post:
items:
keys1:value1
keys2:value2
keys3:value3
keys4:value4
You can do this (Go Playground):
package main
import (
"fmt"
"gopkg.in/yaml.v2"
)
type ItemsStruct struct {
Items map[string]string `yaml:"items"`
}
func main() {
itms := &ItemsStruct{Items: map[string]string{
"keys1": "value1",
"keys2": "value2",
"keys3": "value3",
"keys4": "value4"}}
yamlBytes, err := yaml.Marshal(itms)
if err != nil {
//handle error
}
fmt.Println(string(yamlBytes))
}
And just to add, I see your code is decoding this JSON {"key1":"value1", "key2":"value2", ... } and then encoding it as YAML in your specified format. Here is the Go Playground for that.
I cannot figure out how to make the requests for "access", "clock" and "audit" with the required parameters. Could you give me some guidance on how to make these requests?
package main
import (
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"sync"
"time"
)
const secret = "galumphing"
// Required parameters for the request
type params struct {
Act string
Nonce string
Signature string
Timeout int64
Offset int
}
var (
auditLock sync.Mutex
auditBase int
auditLog []string
)
var (
seenLock sync.Mutex
seenMap = map[string]bool{}
)
var (
clockAsk = make(chan bool)
clockTime = make(chan int64, 1)
)
func main() {
// Endpoints available
http.HandleFunc("/903/access", handleAccess)
http.HandleFunc("/903/clock", handleClock)
http.HandleFunc("/903/audit", handleAudit)
err := http.ListenAndServe(os.Args[1], nil)
if err != nil {
log.Fatal(err)
}
}
func checkCapacity(w http.ResponseWriter) (ok bool) {
auditLock.Lock()
defer auditLock.Unlock()
if len(auditLog) > 10 {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
ok = true
return
}
func audit(r *http.Request, params params, ok bool) {
auditLock.Lock()
defer auditLock.Unlock()
auditLog = append(auditLog, fmt.Sprintf("%v %q %q", ok, r.URL.Path, params.Act))
}
func parse(w http.ResponseWriter, r *http.Request) (params params, ok bool) {
defer func() {
if !ok {
audit(r, params, false)
}
}()
w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Max-Age", "3600")
if r.Method != "POST" {
w.Header().Set("Allow", "POST, OPTIONS")
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusMethodNotAllowed)
}
return
}
err := json.NewDecoder(r.Body).Decode(¶ms)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
log.Printf("%s: %v", r.URL.Path, err)
return
}
h := sha256.New()
fmt.Fprintf(h, "%s\r\n%s\r\n%s\r\n%s", r.URL.Path, params.Act, params.Nonce, secret)
sig := base64.StdEncoding.EncodeToString(h.Sum(nil))
if subtle.ConstantTimeCompare([]byte(sig), []byte(params.Signature)) != 1 {
w.WriteHeader(http.StatusForbidden)
return
}
seenLock.Lock()
seen := seenMap[params.Signature]
if !seen {
seenMap[params.Signature] = true
}
seenLock.Unlock()
if seen {
w.WriteHeader(http.StatusForbidden)
return
}
ok = true
return
}
func handleAccess(w http.ResponseWriter, r *http.Request) {
if !checkCapacity(w) {
return
}
params, ok := parse(w, r)
if !ok {
return
}
switch params.Act {
case "begin":
if params.Timeout < 0 || params.Timeout > 250000 {
w.WriteHeader(http.StatusBadRequest)
audit(r, params, false)
return
}
timer := time.NewTimer(time.Duration(params.Timeout) * time.Microsecond)
select {
case clockAsk <- true: // https://golang.org/ref/spec#Send_statements
w.WriteHeader(http.StatusNoContent)
audit(r, params, true)
case <-timer.C:
w.WriteHeader(http.StatusConflict)
audit(r, params, false)
return
}
go func() {
<-timer.C
select {
case <-clockTime:
default:
}
}()
case "end":
if params.Timeout < 0 {
w.WriteHeader(http.StatusBadRequest)
audit(r, params, false)
return
}
timer := time.NewTimer(time.Duration(params.Timeout) * time.Microsecond)
select {
case value := <-clockTime: // https://golang.org/ref/spec#Receive_operator
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, value)
audit(r, params, true)
case <-timer.C:
w.WriteHeader(http.StatusConflict)
audit(r, params, false)
}
default:
w.WriteHeader(http.StatusBadRequest)
audit(r, params, false)
}
}
func handleClock(w http.ResponseWriter, r *http.Request) {
if !checkCapacity(w) {
return
}
params, ok := parse(w, r)
if !ok {
return
}
switch params.Act {
case "observe":
if params.Timeout != 0 {
w.WriteHeader(http.StatusBadRequest)
audit(r, params, false)
return
}
select {
case <-clockAsk: // https://golang.org/ref/spec#Receive_operator
select {
case clockTime <- time.Now().Unix(): // https://golang.org/ref/spec#Send_statements
default:
}
default:
}
w.WriteHeader(http.StatusNoContent)
audit(r, params, true)
default:
w.WriteHeader(http.StatusBadRequest)
audit(r, params, false)
}
}
func handleAudit(w http.ResponseWriter, r *http.Request) {
params, ok := parse(w, r)
if !ok {
return
}
ok = false
func() {
auditLock.Lock()
defer auditLock.Unlock()
switch params.Act {
case "":
if params.Offset != 0 {
w.WriteHeader(http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, auditBase)
case "burble":
if params.Offset < auditBase || params.Offset > auditBase+len(auditLog) {
w.WriteHeader(http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
for i := params.Offset - auditBase; i < len(auditLog); i++ {
fmt.Fprintf(w, "%d %s\n", auditBase+i, auditLog[i])
}
case "chortle":
if params.Offset > auditBase+len(auditLog) {
w.WriteHeader(http.StatusBadRequest)
return
}
if params.Offset > auditBase {
auditLog = auditLog[params.Offset-auditBase:] // https://golang.org/ref/spec#Slice_expressions
auditBase = params.Offset
}
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusBadRequest)
return
}
ok = true
}()
audit(r, params, ok)
}
There are a few ways to go about this. You can either use a tool like Postman, or simply just use curl from the command line to send a post request to your endpoint. Here is the curl command that I used to hit the endpoint (I am using port 8080 for my testing), the bit after the -d flag is the request body:
curl -X POST http://localhost:8080/903/access -d '{"Act": "act", "Nonce": "nonce", "Signature": "signature", "TimeOut": 64, "Offset": 0}'
I did modify the code to cut out most of the logic. This request just shows an example of how the current code can parse the request body. If you are interested more in how some of the other logic works in the handler functions, let me know and I can edit my response. Here is the code that I used for my testing:
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"os"
)
// Required parameters for the request
type params struct {
Act string
Nonce string
Signature string
Timeout int64
Offset int
}
func main() {
// Endpoints available
http.HandleFunc("/903/access", handleAccess)
err := http.ListenAndServe(os.Args[1], nil)
if err != nil {
log.Fatal(err)
}
}
func parse(w http.ResponseWriter, r *http.Request) (params params, ok bool) {
w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Max-Age", "3600")
if r.Method != "POST" {
w.Header().Set("Allow", "POST, OPTIONS")
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusMethodNotAllowed)
}
return
}
err := json.NewDecoder(r.Body).Decode(¶ms)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
log.Printf("%s: %v", r.URL.Path, err)
return
}
return
}
func handleAccess(w http.ResponseWriter, r *http.Request) {
params, _ := parse(w, r)
fmt.Printf("%+v\n", params)
}
Here is the output that I received when all is said and done:
{Act:act Nonce:nonce Signature:signature Timeout:64 Offset:0}
How can I get all addresses and masks from local interfaces in golang?
I need the actual network mask configured along with every IP address.
This code does not show the network masks in Windows 7:
package main
import (
"fmt"
"log"
"net"
)
func localAddresses() {
ifaces, err := net.Interfaces()
if err != nil {
log.Print(fmt.Errorf("localAddresses: %v\n", err.Error()))
return
}
for _, i := range ifaces {
addrs, err := i.Addrs()
if err != nil {
log.Print(fmt.Errorf("localAddresses: %v\n", err.Error()))
continue
}
for _, a := range addrs {
log.Printf("%v %v\n", i.Name, a)
}
}
}
func main() {
localAddresses()
}
UPDATE: This issue has been fixed in Go: https://github.com/golang/go/issues/5395
There are multiply types of addresses that a net.Interface might have. The Addr is just an interface which may contain a net.IPAddr. But with a type assertion or type switch you can access the actual address type:
package main
import (
"fmt"
"net"
)
func localAddresses() {
ifaces, err := net.Interfaces()
if err != nil {
fmt.Print(fmt.Errorf("localAddresses: %+v\n", err.Error()))
return
}
for _, i := range ifaces {
addrs, err := i.Addrs()
if err != nil {
fmt.Print(fmt.Errorf("localAddresses: %+v\n", err.Error()))
continue
}
for _, a := range addrs {
switch v := a.(type) {
case *net.IPAddr:
fmt.Printf("%v : %s (%s)\n", i.Name, v, v.IP.DefaultMask())
}
}
}
}
func main() {
localAddresses()
}
Edit
Unfortunately the net package doesn't return the Mask of the address. So, you will have to do the low level syscalls that the net package does. The code below is an example, but parsing of the ip and the mask still needs to be done:
package main
import (
"fmt"
"net"
"os"
"syscall"
"unsafe"
)
func getAdapterList() (*syscall.IpAdapterInfo, error) {
b := make([]byte, 1000)
l := uint32(len(b))
a := (*syscall.IpAdapterInfo)(unsafe.Pointer(&b[0]))
// TODO(mikio): GetAdaptersInfo returns IP_ADAPTER_INFO that
// contains IPv4 address list only. We should use another API
// for fetching IPv6 stuff from the kernel.
err := syscall.GetAdaptersInfo(a, &l)
if err == syscall.ERROR_BUFFER_OVERFLOW {
b = make([]byte, l)
a = (*syscall.IpAdapterInfo)(unsafe.Pointer(&b[0]))
err = syscall.GetAdaptersInfo(a, &l)
}
if err != nil {
return nil, os.NewSyscallError("GetAdaptersInfo", err)
}
return a, nil
}
func localAddresses() error {
ifaces, err := net.Interfaces()
if err != nil {
return err
}
aList, err := getAdapterList()
if err != nil {
return err
}
for _, ifi := range ifaces {
for ai := aList; ai != nil; ai = ai.Next {
index := ai.Index
if ifi.Index == int(index) {
ipl := &ai.IpAddressList
for ; ipl != nil; ipl = ipl.Next {
fmt.Printf("%s: %s (%s)\n", ifi.Name, ipl.IpAddress, ipl.IpMask)
}
}
}
}
return err
}
func main() {
err := localAddresses()
if err != nil {
panic(err)
}
}
Some code shamelessly borrowed from interface_windows.go. Even comments are left intact
I'm modifying #ANisus answer and get all interfaces & masks (tested on Windows 10 & WSL in it (Microsoft Ubuntu 16.04.5 LTS):
package main
import (
"fmt"
"net"
)
func localAddresses() {
ifaces, err := net.Interfaces()
if err != nil {
fmt.Print(fmt.Errorf("localAddresses: %+v\n", err.Error()))
return
}
for _, i := range ifaces {
addrs, err := i.Addrs()
if err != nil {
fmt.Print(fmt.Errorf("localAddresses: %+v\n", err.Error()))
continue
}
for _, a := range addrs {
switch v := a.(type) {
case *net.IPAddr:
fmt.Printf("%v : %s (%s)\n", i.Name, v, v.IP.DefaultMask())
case *net.IPNet:
fmt.Printf("%v : %s [%v/%v]\n", i.Name, v, v.IP, v.Mask)
}
}
}
}
func main() {
localAddresses()
}
This should give you the ipnet you're looking for.
ip, ipnet, err := net.ParseCIDR(a.String())
I know this post for Windows 7, but if you use Mac OS X hope this could help you.
Just Call GetNetMask("en0")
func GetNetMask(deviceName string) string {
switch runtime.GOOS {
case "darwin":
cmd := exec.Command("ipconfig", "getoption", deviceName, "subnet_mask")
out, err := cmd.CombinedOutput()
if err != nil {
return ""
}
nm := strings.Replace(string(out), "\n", "", -1)
log.Println("netmask=", nm, " OS=", runtime.GOOS)
return nm
default:
return ""
}
return ""
}
i'm trying to track the upload progress in GOLANG, that's what i got at the moment
func Upload(w http.ResponseWriter, req *http.Request) {
mr, err := req.MultipartReader()
if err != nil {
return
}
for {
// var part *multipart.Part
part, err := mr.NextPart()
mr.partsRead
if err == io.EOF {
break
}
println(part)
}
}
This will work, a stream to calc the bytes read and the total progress you need to point the stream somewhere, in this code example I pointed it to a file.
func Upload(w http.ResponseWriter, req *http.Request) {
mr, err := req.MultipartReader()
if err != nil {
return
}
length := req.ContentLength
for {
part, err := mr.NextPart()
if err == io.EOF {
break
}
var read int64
var p float32
dst, err := os.OpenFile("dstfile", os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return
}
for {
buffer := make([]byte, 100000)
cBytes, err := part.Read(buffer)
if err == io.EOF {
break
}
read = read + int64(cBytes)
//fmt.Printf("read: %v \n",read )
p = float32(read) / float32(length) *100
fmt.Printf("progress: %v \n",p )
dst.Write(buffer[0:cBytes])
}
}
}