I was able to partially solve this with this case
Unfortunately, the Preload() function doesn't seem to work with delving further down in the related object set.
To clarify, I have the following models:
type Room struct {
gorm.Model
Name string
Games []Game `gorm:"ForeignKey:RoomID"`
}
type Game struct {
gorm.Model
RoomID int `gorm:"index"`
Players []Player `gorm:"ForeignKey:GameID"`
}
type Player struct {
gorm.Model
Name string
GameID int `gorm:"index"`
}
When I create a new object with a new Room, Game, and Player object created I get the following data returned (json encoded):
{"Value":{"ID":26,"CreatedAt":"2016-05-15T01:21:22.426234189-07:00","UpdatedAt":"2016-05-15T01:21:22.426234189-07:00","DeletedAt":null,"Name":"foo","Games":[{"ID":17,"CreatedAt":"2016-05-15T01:21:22.427026134-07:00","UpdatedAt":"2016-05-15T01:21:22.427026134-07:00","DeletedAt":null,"RoomID":26,"Turns":null,"Players":[{"ID":4,"CreatedAt":"2016-05-15T01:21:22.427560561-07:00","UpdatedAt":"2016-05-15T01:21:22.427560561-07:00","DeletedAt":null,"Name":"TestPlayer","GameID":17}],"Counter":1,"Assigned":false}],"Testing":false},"Error":null,"RowsAffected":1}
But if I try to query the structure with a preload() function, I get the following:
{"Value":{"ID":26,"CreatedAt":"2016-05-15T01:21:22.426234189-07:00","UpdatedAt":"2016-05-15T01:21:22.426234189-07:00","DeletedAt":null,"Name":"foo","Games":[{"ID":17,"CreatedAt":"2016-05-15T01:21:22.427026134-07:00","UpdatedAt":"2016-05-15T01:21:22.427026134-07:00","DeletedAt":null,"RoomID":26,"Turns":null,"Players":null,"Counter":1,"Assigned":false}],"Testing":false},"Error":null,"RowsAffected":1}
Note that the Players section is now null. Here's my room registration and room query functions:
func RegisterRoom(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
// Handle error
}
// r.PostForm is a map of our POST form values
room := Room{
Name: r.PostFormValue("label"),
Games: []Game{{
Counter: 1,
Players: []Player{{Name: r.PostFormValue("username")}},
}},
}
t := db.Create(&room)
if err := json.NewEncoder(w).Encode(t); err != nil {
panic(err)
}
}
func RoomShow(w http.ResponseWriter, r *http.Request) {
var rm Room
vars := mux.Vars(r)
roomId := vars["roomId"]
id, _ := strconv.Atoi(roomId)
room := db.Preload("Games").First(&rm, id)
result := db.Find(&room)
json.NewEncoder(w).Encode(result)
}
Solved it! Answer was under Nested Preloading smacks forehead
Related
Following up on old post here.
I am iterating over flatProduct.Catalogs slice and populating my productCatalog concurrent map in golang. I am using upsert method so that I can add only unique productID's into my productCatalog map.
Below code is called by multiple go routines in parallel that is why I am using concurrent map here to populate data into it. This code runs in background to populate data in the concurrent map every 30 seconds.
var productRows []ClientProduct
err = json.Unmarshal(byteSlice, &productRows)
if err != nil {
return err
}
for i := range productRows {
flatProduct, err := r.Convert(spn, productRows[i])
if err != nil {
return err
}
if flatProduct.StatusCode == definitions.DONE {
continue
}
r.products.Set(strconv.Itoa(flatProduct.ProductId, 10), flatProduct)
for _, catalogId := range flatProduct.Catalogs {
catalogValue := strconv.FormatInt(int64(catalogId), 10)
r.productCatalog.Upsert(catalogValue, flatProduct.ProductId, func(exists bool, valueInMap interface{}, newValue interface{}) interface{} {
productID := newValue.(int64)
if valueInMap == nil {
return map[int64]struct{}{productID: {}}
}
oldIDs := valueInMap.(map[int64]struct{})
// value is irrelevant, no need to check if key exists
// I think problem is here
oldIDs[productID] = struct{}{}
return oldIDs
})
}
}
And below are my getters in the same class where above code is there. These getters are used by main application threads to get data from the map or get the whole map.
func (r *clientRepository) GetProductMap() *cmap.ConcurrentMap {
return r.products
}
func (r *clientRepository) GetProductCatalogMap() *cmap.ConcurrentMap {
return r.productCatalog
}
func (r *clientRepository) GetProductData(pid string) *definitions.FlatProduct {
pd, ok := r.products.Get(pid)
if ok {
return pd.(*definitions.FlatProduct)
}
return nil
}
This is how I am reading data from this productCatalog cmap but my system is crashing on the below range statement -
// get productCatalog map which was populated above
catalogProductMap := clientRepo.GetProductCatalogMap()
productIds, ok := catalogProductMap.Get("211")
data, _ := productIds.(map[int64]struct{})
// I get panic here after sometime
for _, pid := range data {
...
}
Error I am getting as - fatal error: concurrent map iteration and map write.
I think issue is r.productCatalog is a concurrentmap, but oldIDs[productID] is a normal map which is causing issues while I am iterating in the for loop above.
How can I fix this race issue I am seeing? One way I can think of is making oldIDs[productID] as concurrent map but if I do that approach then my memory increase by a lot and eventually goes OOM. Below is what I have tried which works and it solves the race condition but it increases the memory by a lot which is not what I want -
r.productCatalog.Upsert(catalogValue, flatProduct.ProductId, func(exists bool, valueInMap interface{}, newValue interface{}) interface{} {
productID := newValue.(int64)
if valueInMap == nil {
// return map[int64]struct{}{productID: {}}
return cmap.New()
}
// oldIDs := valueInMap.(map[int64]struct{})
oldIDs := valueInMap.(cmap.ConcurrentMap)
// value is irrelevant, no need to check if key exists
// oldIDs[productID] = struct{}{}
oldIDs.Set(strconv.FormatInt(productID, 10), struct{}{})
return oldIDs
})
Any other approach I can do which doesn't increase memory and also fixes the race condition I am seeing?
Note
I am still using v1 version of cmap without generics and it deals with strings as keys.
Rather than a plain map[int64]struct{} type, you could define a struct which holds the map and a mutex to control the access to the map:
type myMap struct{
m sync.Mutex
data map[int64]struct{}
}
func (m *myMap) Add(productID int64) {
m.m.Lock()
defer m.m.Unlock()
m.data[productID] = struct{}{}
}
func (m *myMap) List() []int64 {
m.m.Lock()
defer m.m.Unlock()
var res []int64
for id := range m.data {
res = append(res, id)
}
// sort slice if you need
return res
}
With the sample implementation above, you would have to be careful to store *myMap pointers (as opposed to plain myMap structs) in your cmap.ConcurrentMap structure.
I'm trying to append an id (and other info) to the url, so I can access it later, but I can't find the right method after some research.
I've tried to use Get() method, query(), Add(), but I couldn't redirect the URL.
var email_ployer string
func RegisterNewPloyer(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/ployer/register" {
http.Error(w, "404 not found.", http.StatusNotFound)
return
}
db := connect.ConnectDB()
defer db.Close()
switch r.Method {
case "POST":
email_ployer = r.FormValue("email")
senha := r.FormValue("senha")
senha, _ = HashPassword(senha)
tx, _ := db.Begin()
stmt, _ := tx.Prepare("INSERT INTO ployers(email_ployer, senha_ployer) VALUES(?,?)")
_, erro := stmt.Exec(email_ployer, senha)
if erro != nil {
tx.Rollback()
log.Fatal(erro)
}
tx.Commit()
}
Redirect(w, r)
}
func Redirect(w http.ResponseWriter, r *http.Request) {
db2 := connect.ConnectDB()
defer db2.Close()
var id string
tx, _ := db2.Begin()
rows, _ := tx.Query("SELECT id FROM ployers WHERE email_ployer = '?'", email_ployer)
for rows.Next() {
if err := rows.Scan(&id); err != nil {
log.Fatal(err)
}
if err := rows.Err(); err != nil {
log.Fatal(err)
}
}
http.Redirect(w, r, x, http.StatusSeeOther)
}
func main() {
http.HandleFunc("/ployer/seja-um-de-nos", LoadPloyerContent)
http.HandleFunc("/ployer/register", register.RegisterNewPloyer)
http.HandleFunc("/ployer/complete/", LoadPloyerContent)
http.HandleFunc("/ployer/register-received", LoadPloyerContent)
log.Fatal(http.ListenAndServe(":3306", nil))
}
In my system, I want the user to register his E-mail and password, create an new user in the DB and redirect the URL to something like localhost:3306/ployer/complete/id
Just use res.LastInsertId() to get the id and pass it to your redirect function, and build the url:
func RegisterNewPloyer(w http.ResponseWriter, r *http.Request) {
//...
//...
res, erro := stmt.Exec(email_ployer, senha)
if erro != nil {
tx.Rollback()
log.Fatal(erro)
}
tx.Commit()
id, erro := res.LastInsertId()
if erro != nil {
log.Fatal(erro)
}
Redirect(w, r, id)
}
func Redirect(w http.ResponseWriter, r *http.Request, id int64) {
uri := fmt.Sprintf("http://localhost:3306/ployer/complete/%d", id)
http.Redirect(w, r, uri, http.StatusSeeOther)
}
You should use url.Values to construct a map of query string values. Later on, you can use Encode method to generate encoded query string. Assign this value to Url.RawQuery to get the final output.
e.g.:
base, _ := url.Parse("https://www.foo.com/")
// Create an instance of Values to hold your query string parameters
values := url.Values{}
values.Add("abc", "def")
// Set the encoded output to RawQuery attribute of Url
base.RawQuery = values.Encode()
println(base.String())
If you are starting with a URL that already has query string parameters and you want add/modify items in it, use Query() method to obtain a reference to parsed map.
e.g.:
base, _ := url.Parse("https://www.foo.com/?a=b")
values := base.Query()
values.Set("a", "c")
values.Set("x", "y")
// Set the encoded output to RawQuery attribute of Url
base.RawQuery = values.Encode()
println(base.String())
I am getting data from source A and storing it in a slice of structs like so:
type ProductPrice struct {
Type string
Sku string
UnitPrice string
PriceList string
standardPrice string
specialPrice string
specialStart string
specialEnd string
pricingUnit string
categoryCode string
isOnSpecial bool
}
func getProductPricesFromDatabase(instance string) []ProductPrice {
rows, err := myDBConnection.Query(// My query here)
if err != nil {
log.Fatal("There was an issue with the query for product price: ", err)
}
defer rows.Close()
var productPrices []ProductPrice
for rows.Next() {
var product = ProductPrice{}
err := rows.Scan(
&product.Type,
&product.Sku,
&product.standardPrice,
&product.specialPrice,
&product.specialStart,
&product.specialEnd,
&product.pricingUnit,
&product.PriceList,
&product.categoryCode,
)
if err != nil {
log.Fatal("product price scan error: ", err)
}
productPrices = append(productPrices, product)
}
return productPrices
}
I am then getting some data from source B and storing it in a slice of structs like so:
type ContractProductPrice struct {
CustID string
PriceBy string
AppliesTo string
PriceList string
StartDate string
EndDate string
PricingAdjustmentType string
PricingAdjustmentValue string
UseLowest string
}
func getContractProductPricesFromDatabase(instance string) []ContractProductPrice {
rows, err := myDBConnection.Query(// My query here)
if err != nil {
log.Fatal("There was an issue with the query for contract product price: ", err)
}
defer rows.Close()
var contractProductPrices []ContractProductPrice
for rows.Next() {
var product = ContractProductPrice{}
err := rows.Scan(
&product.CustID,
&product.PriceBy,
&product.AppliesTo,
&product.PriceList,
&product.StartDate,
&product.EndDate,
&product.PricingAdjustmentType,
&product.PricingAdjustmentValue,
&product.UseLowest,
)
if err != nil {
log.Fatal("contract product price scan error: ", err)
}
contractProductPrices = append(contractProductPrices, product)
}
return contractProductPrices
}
After getting the data from source B, I am wanting to update the slice of structs from source A with some data from source B.
productPrices := getProductPricesFromDatabase(instance)
contractProductPrices := getContractProductPricesFromDatabase(instance)
processedProductPrices := processProductPricesFromDatabase(productPrices, contractProductPrices)
func processProductPricesFromDatabase(productPrices []ProductPrice, contractProductPrices []ContractProductPrice) []ProductPrice {
// Loop over contact prices and update relevant product prices
for _, contractPrice := range contractProductPrices {
for _, product := range productPrices {
if contractPrice.AppliesTo == product.Sku {
product.UnitPrice = contractPrice.PricingAdjustmentValue
}
}
}
return productPrices
}
However, after this runs, the unit prices in processedProductPrices is still empty.
From my searching, I understand what the issue is; Go passes by value and so I am not updating the original memory address and so the values are not changing.
However, I do not understand/know what I need to change to fix this given I am working with a slice of structs rather than a simpler example of a slice of number/strings etc.
How can I update productPrices so that when I return it, processedProductPrices is equal to the updated productPrices slice of structs?
Anytime you're dealing with values that you know you'll need to modify, it is best, at least in my opinion, to use pointers. They'll make your life easier.
So instead of:
func getProductPricesFromDatabase(instance string) []ProductPrice {
// ...
var productPrices []ProductPrice
for rows.Next() {
var product = ProductPrice{}
// ...
}
return productPrices
}
I would recommend you refactor your code to:
func getProductPricesFromDatabase(instance string) []*ProductPrice {
// ...
var productPrices []*ProductPrice
for rows.Next() {
var product = new(ProductPrice)
// ...
}
return productPrices
}
Now do the same with getContractProductPricesFromDatabase and finally update the argument types to your processProductPricesFromDatabase function:
func processProductPricesFromDatabase(productPrices []*ProductPrice, contractProductPrices []*ContractProductPrice) []*ProductPrice {
// Loop over contact prices and update relevant product prices
for _, contractPrice := range contractProductPrices {
for _, product := range productPrices {
if contractPrice.AppliesTo == product.Sku {
product.UnitPrice = contractPrice.PricingAdjustmentValue
}
}
}
return productPrices
}
As an alternative, if you want to keep using non-pointer types, you can directly modify the values referenced by the slice by indexing into it.
func processProductPricesFromDatabase(productPrices []ProductPrice, contractProductPrices []ContractProductPrice) []ProductPrice {
// Loop over contact prices and update relevant product prices
for _, contractPrice := range contractProductPrices {
for i, _ := range productPrices {
if contractPrice.AppliesTo == productPrices[i].Sku {
productPrices[i].UnitPrice = contractPrice.PricingAdjustmentValue
}
}
}
return productPrices
}
I am developing a program in go that needs to store and retrieve an array of custom structure instances using the sessions package of the Gorilla toolkit. For restoring the custom structure I need to make use of reflection features. The issue is that my structure named Timestamp includes two time.Time instances and I have not been able to restore the instances. Thus, my question is how to restore a time.Time instance.
Below you can see my code for the Timespan structure as well as the code for storing and reading the Timespan array in the session store.
type Timespan struct {
ID uint8;
StartDate time.Time;
EndDate time.Time;
}
func (server *WebServer) setTimespans(writer http.ResponseWriter, request *http.Request, timespans [model.TimespanCount]*model.Timespan) error {
var session *sessions.Session;
var sessionDecodingException error;
session, sessionDecodingException = server.SessionStore.Get(request, authenticationSessionName);
if sessionDecodingException != nil {
return sessionDecodingException;
}
session.Values[sessionTimestamps] = timespans;
return nil;
}
func (server *WebServer) getTimespans(request *http.Request) ([model.TimespanCount]*model.Timespan, error) {
var session *sessions.Session;
var sessionDecodingException error;
session, sessionDecodingException = server.SessionStore.Get(request, authenticationSessionName);
var readTimespans [model.TimespanCount]*model.Timespan;
if sessionDecodingException != nil {
return readTimespans, sessionDecodingException;
}
interfaceValue := reflect.ValueOf(session.Values[sessionTimestamps]);
var actuallyAddedTimespan *model.Timespan;
for counter := 0; counter < model.TimespanCount; counter++ {
actuallyAddedTimespan = &model.Timespan{};
actuallyReflectedTimespan := interfaceValue.Index(counter).Elem();
actuallyAddedTimespan.ID = uint8(actuallyReflectedTimespan.FieldByName("ID").Uint());
//actuallyAddedTimespan.StartDate = actuallyReflectedTimespan.FieldByName("StartDate");
//actuallyAddedTimespan.EndDate = actuallyReflectedTimespan.FieldByName("EndDate");
fmt.Println(actuallyAddedTimespan);
}
return readTimespans, nil;
}
You need to get the interface to the field:
actuallyAddedTimespan.StartDate = actuallyReflectedTimespan.FieldByName("StartDate").Interface().(time.Time)
playground
Personal opinion time, using reflection for this rather than using a simple interface is:
Slow.
Inefficient
Can break easily if you change how your struct looks and forget to update the reflection code.
Example of using an interface:
func main() {
ts := &Timespan{ID: 102, StartDate: time.Now().AddDate(6, 0, 0), EndDate: time.Now().AddDate(8, 0, 0)}
m := map[string]interface{}{
"key": ts,
}
switch v := m["key"].(type) {
case Timespaner:
fmt.Println(v.Value())
default:
fmt.Println("wtfmate?")
}
}
func (ts *Timespan) Value() (id uint8, start, end time.Time) {
return ts.ID, ts.StartDate, ts.EndDate
}
type Timespaner interface {
Value() (id uint8, start, end time.Time)
}
playground
You can use the Interface() function to get the interface{} value from a struct. Then, you can use a type assertion to get the correct type back:
func main() {
t := &Timespan{42, time.Now(), time.Now()}
reflectedPointer := reflect.ValueOf(t)
reflectedTimespan := reflectedPointer.Elem()
var timespan Timespan = reflectedTimespan.Interface().(Timespan)
fmt.Println(*t)
fmt.Println(timespan)
}
I want to convert a struct to map in Golang. It would also be nice if I could use the JSON tags as keys in the created map (otherwise defaulting to field name).
Edit Dec 14, 2020
Since structs repo was archived, you can use mapstructure instead.
Edit TL;DR version, Jun 15, 2015
If you want the fast solution for converting a structure to map, see the accepted answer, upvote it and use that package.
Happy coding! :)
Original Post
So far I have this function, I am using the reflect package but I don't understand well how to use the package, please bear with me.
func ConvertToMap(model interface{}) bson.M {
ret := bson.M{}
modelReflect := reflect.ValueOf(model)
if modelReflect.Kind() == reflect.Ptr {
modelReflect = modelReflect.Elem()
}
modelRefType := modelReflect.Type()
fieldsCount := modelReflect.NumField()
var fieldData interface{}
for i := 0; i < fieldsCount; i++ {
field := modelReflect.Field(i)
switch field.Kind() {
case reflect.Struct:
fallthrough
case reflect.Ptr:
fieldData = ConvertToMap(field.Interface())
default:
fieldData = field.Interface()
}
ret[modelRefType.Field(i).Name] = fieldData
}
return ret
}
Also I looked at JSON package source code, because it should contain my needed implementation (or parts of it) but don't understand too much.
I also had need for something like this. I was using an internal package which was converting a struct to a map. I decided to open source it with other struct based high level functions. Have a look:
https://github.com/fatih/structs
It has support for:
Convert struct to a map
Extract the fields of a struct to a []string
Extract the values of a struct to a []values
Check if a struct is initialized or not
Check if a passed interface is a struct or a pointer to struct
You can see some examples here: http://godoc.org/github.com/fatih/structs#pkg-examples
For example converting a struct to a map is a simple:
type Server struct {
Name string
ID int32
Enabled bool
}
s := &Server{
Name: "gopher",
ID: 123456,
Enabled: true,
}
// => {"Name":"gopher", "ID":123456, "Enabled":true}
m := structs.Map(s)
The structs package has support for anonymous (embedded) fields and nested structs. The package provides to filter certain fields via field tags.
From struct to map[string]interface{}
package main
import (
"fmt"
"encoding/json"
)
type MyData struct {
One int
Two string
Three int
}
func main() {
in := &MyData{One: 1, Two: "second"}
var inInterface map[string]interface{}
inrec, _ := json.Marshal(in)
json.Unmarshal(inrec, &inInterface)
// iterate through inrecs
for field, val := range inInterface {
fmt.Println("KV Pair: ", field, val)
}
}
go playground here
Here is a function I've written in the past to convert a struct to a map, using tags as keys
// ToMap converts a struct to a map using the struct's tags.
//
// ToMap uses tags on struct fields to decide which fields to add to the
// returned map.
func ToMap(in interface{}, tag string) (map[string]interface{}, error){
out := make(map[string]interface{})
v := reflect.ValueOf(in)
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
// we only accept structs
if v.Kind() != reflect.Struct {
return nil, fmt.Errorf("ToMap only accepts structs; got %T", v)
}
typ := v.Type()
for i := 0; i < v.NumField(); i++ {
// gets us a StructField
fi := typ.Field(i)
if tagv := fi.Tag.Get(tag); tagv != "" {
// set key of map to value in struct field
out[tagv] = v.Field(i).Interface()
}
}
return out, nil
}
Runnable example here.
Note, if you have multiple fields with the same tag value, then you will obviously not be able to store them all within a map. It might be prudent to return an error if that happens.
I like the importable package for the accepted answer, but it does not translate my json aliases. Most of my projects have a helper function/class that I import.
Here is a function that solves my specific problem.
// Converts a struct to a map while maintaining the json alias as keys
func StructToMap(obj interface{}) (newMap map[string]interface{}, err error) {
data, err := json.Marshal(obj) // Convert to a json string
if err != nil {
return
}
err = json.Unmarshal(data, &newMap) // Convert to a map
return
}
And in the main, this is how it would be called...
package main
import (
"fmt"
"encoding/json"
"github.com/fatih/structs"
)
type MyStructObject struct {
Email string `json:"email_address"`
}
func main() {
obj := &MyStructObject{Email: "test#test.com"}
// My solution
fmt.Println(StructToMap(obj)) // prints {"email_address": "test#test.com"}
// The currently accepted solution
fmt.Println(structs.Map(obj)) // prints {"Email": "test#test.com"}
}
package main
import (
"fmt"
"reflect"
)
type bill struct {
N1 int
N2 string
n3 string
}
func main() {
a := bill{4, "dhfthf", "fdgdf"}
v := reflect.ValueOf(a)
values := make(map[string]interface{}, v.NumField())
for i := 0; i < v.NumField(); i++ {
if v.Field(i).CanInterface() {
values[v.Type().Field(i).Name] = v.Field(i).Interface()
} else {
fmt.Printf("sorry you have a unexported field (lower case) value you are trying to sneak past. I will not allow it: %v\n", v.Type().Field(i).Name)
}
}
fmt.Println(values)
passObject(&values)
}
func passObject(v1 *map[string]interface{}) {
fmt.Println("yoyo")
}
I'm a bit late but I needed this kind of feature so I wrote this. Can resolve nested structs. By default, uses field names but can also use custom tags. A side effect is that if you set the tagTitle const to json, you could use the json tags you already have.
package main
import (
"fmt"
"reflect"
)
func StructToMap(val interface{}) map[string]interface{} {
//The name of the tag you will use for fields of struct
const tagTitle = "kelvin"
var data map[string]interface{} = make(map[string]interface{})
varType := reflect.TypeOf(val)
if varType.Kind() != reflect.Struct {
// Provided value is not an interface, do what you will with that here
fmt.Println("Not a struct")
return nil
}
value := reflect.ValueOf(val)
for i := 0; i < varType.NumField(); i++ {
if !value.Field(i).CanInterface() {
//Skip unexported fields
continue
}
tag, ok := varType.Field(i).Tag.Lookup(tagTitle)
var fieldName string
if ok && len(tag) > 0 {
fieldName = tag
} else {
fieldName = varType.Field(i).Name
}
if varType.Field(i).Type.Kind() != reflect.Struct {
data[fieldName] = value.Field(i).Interface()
} else {
data[fieldName] = StructToMap(value.Field(i).Interface())
}
}
return data
}
map := Structpb.AsMap()
// map is the map[string]interface{}