I have a hot code path (100k runs) that optionally needs to fill a map with some values.
I am trying to optimize it and one of the things that can help would be lazy memory initializtion.
my current code is
mystruct struct {
mymap
}
for i range 10000 {
mystruct = {}
mystruct.mymap = map[string]string
if variable_exists {
mystruct.mymap[variable] = blah
}
}
I then later use this variable in a ton of range statements so it has to be initialized.
but the vast majority of the time the map just lays empty so it would be nice if i did not need to initialize a map just to leave it empty.
so my hope would be something like
mystruct struct {
mymap
}
default_map = make(map[string]string)
for i range 10000 {
mystruct = {}
mystruct.mymap = default_map
if variable_exists {
if mystruct.mymap == default_map {
mystruct.mymap = make(map[string]string)
}
mystruct.mymap[variable] = blah
}
}
but that does not work.
I found deepequal but that would be wayyyyyy too slow.
most languages allow checks to see if they are pointed at the same object so how do I do that in go?
Initialize the map field with nil. Compare the field to nil before adding a value to the map:
for i := 0; i < 10000; i++ {
var m mystruct
if variable_exists {
if m.mymap == nil {
m.mymap = make(map[string]string)
}
m.mymap[variable] = blah
}
}
Related
I have a below struct where I have a nested map for CustomersIndex which allocates bunch of internal maps causing memory increase. I profiled it so I noticed this. I am trying to see if there is any way to redesign my CustomersIndex data structure which doesn't uses nested map?
const (
departmentsKey = "departments"
)
type CustomerManifest struct {
Customers []definitions.Customer
CustomersIndex map[int]map[int]definitions.Customer
}
This is the way it is being populated here in my below code:
func updateData(mdmCache *mdm.Cache) map[string]interface{} {
memCache := mdmCache.MemCache()
var customers []definitions.Customer
var customersIndex = map[int]map[int]definitions.Customer{}
for _, r := range memCache.Customer {
customer := definitions.Customer{
Id: int(r.Id),
SetId: int(r.DepartmentSetId),
}
customers = append(customers, customer)
_, yes := customersIndex[customer.SetId]
if !yes {
customersIndex[customer.SetId] = make(map[int]definitions.Customer)
}
customersIndex[customer.SetId][customer.Id] = customer
}
return map[string]interface{}{
departmentsKey: &CustomerManifest{Customers: customers, CustomersIndex: customersIndex},
}
}
And this is the way I am getting my CustomersIndex nested map.
func (c *Client) GetCustomerIndex() map[int]map[int]definitions.Customer {
c.mutex.RLock()
defer c.mutex.RUnlock()
customersIndex := c.data[departmentsKey].(*CustomerManifest).CustomersIndex
return customersIndex
}
Is there any way to design my CustomersIndex in a way where I don't have to use nested map?
You don't need to allocate a map until you put values in it.
type CustomerManifest struct {
Customers []definitions.Customer
CustomersIndex map[int]map[int]definitions.Customer
}
func (m *CustomerManifest) AddCustomerDefinition(x, y int, customer definitions.Customer) {
// Get the existing map, if exists.
innerMap := m.CustomersIndex[x]
// If it doesn't exist, allocate it.
if innerMap == nil {
innerMap = make(map[int]definitions.Customer)
m.CustomersIndex[x] = innerMap
}
// Add the value to the inner map, which now exists.
innerMap[y] = customer
}
I am trying to convert some Objective C code provided in one of Apple's code examples here: https://developer.apple.com/library/mac/samplecode/avsubtitleswriterOSX/Listings/avsubtitleswriter_SubtitlesTextReader_m.html
The result I have come up with thus far is as follows:
func copySampleBuffer() -> CMSampleBuffer? {
var textLength : Int = 0
var sampleSize : Int = 0
if (text != nil) {
textLength = text!.characters.count
sampleSize = text!.lengthOfBytesUsingEncoding(NSUTF16StringEncoding)
}
var sampleData = [UInt8]()
// Append text length
sampleData.append(UInt16(textLength).hiByte())
sampleData.append(UInt16(textLength).loByte())
// Append the text
for char in (text?.utf16)! {
sampleData.append(char.bigEndian.hiByte())
sampleData.append(char.bigEndian.loByte())
}
if (self.forced) {
// TODO
}
let samplePtr = UnsafeMutablePointer<[UInt8]>.alloc(1)
samplePtr.memory = sampleData
var sampleTiming = CMSampleTimingInfo()
sampleTiming.duration = self.timeRange.duration;
sampleTiming.presentationTimeStamp = self.timeRange.start;
sampleTiming.decodeTimeStamp = kCMTimeInvalid;
let formatDescription = copyFormatDescription()
let dataBufferUMP = UnsafeMutablePointer<Optional<CMBlockBuffer>>.alloc(1)
CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, samplePtr, sampleSize, kCFAllocatorMalloc, nil, 0, sampleSize, 0, dataBufferUMP);
let sampleBufferUMP = UnsafeMutablePointer<Optional<CMSampleBuffer>>.alloc(1)
CMSampleBufferCreate(kCFAllocatorDefault, dataBufferUMP.memory, true, nil, nil, formatDescription, 1, 1, &sampleTiming, 1, &sampleSize, sampleBufferUMP);
let sampleBuffer = sampleBufferUMP.memory
sampleBufferUMP.destroy()
sampleBufferUMP.dealloc(1)
dataBufferUMP.destroy()
dataBufferUMP.dealloc(1)
samplePtr.destroy()
//Crash if I call dealloc here
//Error is: error for object 0x10071e400: pointer being freed was not allocated
//samplePtr.dealloc(1)
return sampleBuffer;
}
I would like to avoid the "Unsafe*" types where possible, though I am not sure it is possible here. I also looked at using a struct and then somehow seeing to pack it somehow, but example I see seem to be based of sizeof, which uses the size of the definition, rather than the current size of the structure. This would have been the structure I would have used:
struct SubtitleAtom {
var length : UInt16
var text : [UInt16]
var forced : Bool?
}
Any advice on most suitable Swift 2 code for this function would be appreciated.
so, at first, you code use this pattern
class C { deinit { print("I got deinit'd!") } }
struct S { var objectRef:AnyObject? }
func foo() {
let ptr = UnsafeMutablePointer<S>.alloc(1)
let o = C()
let fancy = S(objectRef: o)
ptr.memory = fancy
ptr.destroy() //deinit runs here!
ptr.dealloc(1) //don't leak memory
}
// soon or later this code should crash :-)
(1..<1000).forEach{ i in
foo()
print(i)
}
Try it in a playground and most likely it crash :-). What's wrong with it? The trouble is your unbalanced retain / release cycles. How to write the same in the safe manner? You removed dealloc part. But try to do it in my snippet and see the result. The code crash again :-). The only safe way is to properly initialize and de-ininitialize (destroy) the underlying ptr's Memory as you can see in next snippet
class C { deinit { print("I got deinit'd!") } }
struct S { var objectRef:AnyObject? }
func foo() {
let ptr = UnsafeMutablePointer<S>.alloc(1)
let o = C()
let fancy = S(objectRef: o)
ptr.initialize(fancy)
ptr.destroy()
ptr.dealloc(1)
}
(1..<1000).forEach{ i in
foo()
print(i)
}
Now the code is executed as expected and all retain / release cycles are balanced.
I have a slice that I want to change (for example i want to remove the first element) using a function. I thought to use a pointer, but I still can't index it. What am I doing wrong?
Playground link:
func change(list *[]int) {
fmt.Println(*list)
*list = *list[1:] //This line screws everything up
}
var list = []int{1, 2, 3}
func main() {
change(&list)
}
You need to use (*list).
func change(list *[]int) {
*list = (*list)[1:]
}
or a different approach that's usually more go idomatic:
func change(list []int) []int {
return list[1:]
}
playground
I have a static func that creates a dictionary from a bunch of enums in a struct.
It looks like this:
// Struct section starts here
struct Card {
var rank: Rank
var suit: Suit
func simpleDescription() -> String {
return "The \(rank.simpleDescription()) of \(suit.simpleDescription())"
}
// Must use static for non-instance methods
static func createDeck() -> [Card] {
var deck = [Card]()
var n = 1
while let rank = Rank.fromRaw(n) {
var m = 1
while let suit = Suit.fromRaw(m) {
deck += Card(rank: rank, suit: suit)
m++
}
n++
}
return deck
}
}
The output is a nice bunch of enums when I call Card.createDeck() that look like this
{(Enum Value), (Enum Value)}
...
Lots of this
...
{(Enum Value), (Enum Value)}
I can even see the wrapped values if I do this:
for n in deck {
println(n)
}
Which nets me lots of this:
_TtV13__lldb_expr_04Card
I can't figure out how to unwrap these enums; please can someone help me?
Thanks
First of all: you have an array of card, not a dictionary.
Second: items in the array are not optionals hence no need to unwrap anything.
In for loop you can access card description:
for n in deck {
println(n.simpleDescription())
}
I want to convert a struct to map in Golang. It would also be nice if I could use the JSON tags as keys in the created map (otherwise defaulting to field name).
Edit Dec 14, 2020
Since structs repo was archived, you can use mapstructure instead.
Edit TL;DR version, Jun 15, 2015
If you want the fast solution for converting a structure to map, see the accepted answer, upvote it and use that package.
Happy coding! :)
Original Post
So far I have this function, I am using the reflect package but I don't understand well how to use the package, please bear with me.
func ConvertToMap(model interface{}) bson.M {
ret := bson.M{}
modelReflect := reflect.ValueOf(model)
if modelReflect.Kind() == reflect.Ptr {
modelReflect = modelReflect.Elem()
}
modelRefType := modelReflect.Type()
fieldsCount := modelReflect.NumField()
var fieldData interface{}
for i := 0; i < fieldsCount; i++ {
field := modelReflect.Field(i)
switch field.Kind() {
case reflect.Struct:
fallthrough
case reflect.Ptr:
fieldData = ConvertToMap(field.Interface())
default:
fieldData = field.Interface()
}
ret[modelRefType.Field(i).Name] = fieldData
}
return ret
}
Also I looked at JSON package source code, because it should contain my needed implementation (or parts of it) but don't understand too much.
I also had need for something like this. I was using an internal package which was converting a struct to a map. I decided to open source it with other struct based high level functions. Have a look:
https://github.com/fatih/structs
It has support for:
Convert struct to a map
Extract the fields of a struct to a []string
Extract the values of a struct to a []values
Check if a struct is initialized or not
Check if a passed interface is a struct or a pointer to struct
You can see some examples here: http://godoc.org/github.com/fatih/structs#pkg-examples
For example converting a struct to a map is a simple:
type Server struct {
Name string
ID int32
Enabled bool
}
s := &Server{
Name: "gopher",
ID: 123456,
Enabled: true,
}
// => {"Name":"gopher", "ID":123456, "Enabled":true}
m := structs.Map(s)
The structs package has support for anonymous (embedded) fields and nested structs. The package provides to filter certain fields via field tags.
From struct to map[string]interface{}
package main
import (
"fmt"
"encoding/json"
)
type MyData struct {
One int
Two string
Three int
}
func main() {
in := &MyData{One: 1, Two: "second"}
var inInterface map[string]interface{}
inrec, _ := json.Marshal(in)
json.Unmarshal(inrec, &inInterface)
// iterate through inrecs
for field, val := range inInterface {
fmt.Println("KV Pair: ", field, val)
}
}
go playground here
Here is a function I've written in the past to convert a struct to a map, using tags as keys
// ToMap converts a struct to a map using the struct's tags.
//
// ToMap uses tags on struct fields to decide which fields to add to the
// returned map.
func ToMap(in interface{}, tag string) (map[string]interface{}, error){
out := make(map[string]interface{})
v := reflect.ValueOf(in)
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
// we only accept structs
if v.Kind() != reflect.Struct {
return nil, fmt.Errorf("ToMap only accepts structs; got %T", v)
}
typ := v.Type()
for i := 0; i < v.NumField(); i++ {
// gets us a StructField
fi := typ.Field(i)
if tagv := fi.Tag.Get(tag); tagv != "" {
// set key of map to value in struct field
out[tagv] = v.Field(i).Interface()
}
}
return out, nil
}
Runnable example here.
Note, if you have multiple fields with the same tag value, then you will obviously not be able to store them all within a map. It might be prudent to return an error if that happens.
I like the importable package for the accepted answer, but it does not translate my json aliases. Most of my projects have a helper function/class that I import.
Here is a function that solves my specific problem.
// Converts a struct to a map while maintaining the json alias as keys
func StructToMap(obj interface{}) (newMap map[string]interface{}, err error) {
data, err := json.Marshal(obj) // Convert to a json string
if err != nil {
return
}
err = json.Unmarshal(data, &newMap) // Convert to a map
return
}
And in the main, this is how it would be called...
package main
import (
"fmt"
"encoding/json"
"github.com/fatih/structs"
)
type MyStructObject struct {
Email string `json:"email_address"`
}
func main() {
obj := &MyStructObject{Email: "test#test.com"}
// My solution
fmt.Println(StructToMap(obj)) // prints {"email_address": "test#test.com"}
// The currently accepted solution
fmt.Println(structs.Map(obj)) // prints {"Email": "test#test.com"}
}
package main
import (
"fmt"
"reflect"
)
type bill struct {
N1 int
N2 string
n3 string
}
func main() {
a := bill{4, "dhfthf", "fdgdf"}
v := reflect.ValueOf(a)
values := make(map[string]interface{}, v.NumField())
for i := 0; i < v.NumField(); i++ {
if v.Field(i).CanInterface() {
values[v.Type().Field(i).Name] = v.Field(i).Interface()
} else {
fmt.Printf("sorry you have a unexported field (lower case) value you are trying to sneak past. I will not allow it: %v\n", v.Type().Field(i).Name)
}
}
fmt.Println(values)
passObject(&values)
}
func passObject(v1 *map[string]interface{}) {
fmt.Println("yoyo")
}
I'm a bit late but I needed this kind of feature so I wrote this. Can resolve nested structs. By default, uses field names but can also use custom tags. A side effect is that if you set the tagTitle const to json, you could use the json tags you already have.
package main
import (
"fmt"
"reflect"
)
func StructToMap(val interface{}) map[string]interface{} {
//The name of the tag you will use for fields of struct
const tagTitle = "kelvin"
var data map[string]interface{} = make(map[string]interface{})
varType := reflect.TypeOf(val)
if varType.Kind() != reflect.Struct {
// Provided value is not an interface, do what you will with that here
fmt.Println("Not a struct")
return nil
}
value := reflect.ValueOf(val)
for i := 0; i < varType.NumField(); i++ {
if !value.Field(i).CanInterface() {
//Skip unexported fields
continue
}
tag, ok := varType.Field(i).Tag.Lookup(tagTitle)
var fieldName string
if ok && len(tag) > 0 {
fieldName = tag
} else {
fieldName = varType.Field(i).Name
}
if varType.Field(i).Type.Kind() != reflect.Struct {
data[fieldName] = value.Field(i).Interface()
} else {
data[fieldName] = StructToMap(value.Field(i).Interface())
}
}
return data
}
map := Structpb.AsMap()
// map is the map[string]interface{}