From 29f139d4245aea7f1fd20ce5b9f634fbbe9fb7a0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 25 Sep 2017 22:48:46 +0000 Subject: [PATCH 1/4] Re-implement LatestMap as a sorted slice for better performance both the Python generator and the Go generated code are checked in --- extras/generate_latest_map | 187 +++++++++++---- report/latest_map_generated.go | 365 ++++++++++++++++++++++------- report/latest_map_internal_test.go | 2 +- 3 files changed, 417 insertions(+), 137 deletions(-) diff --git a/extras/generate_latest_map b/extras/generate_latest_map index f55ce67c48..2e7920cf13 100755 --- a/extras/generate_latest_map +++ b/extras/generate_latest_map @@ -19,11 +19,12 @@ function generate_header() { package report import ( + "bytes" "fmt" + "sort" "time" "github.com/ugorji/go/codec" - "github.com/weaveworks/ps" ) EOF } @@ -35,7 +36,6 @@ function generate_latest_map() { local lowercase_data_type="${data_type,}" local entry_type="${lowercase_data_type}LatestEntry" local latest_map_type="${uppercase_data_type}LatestMap" - local empty_latest_map_variable="empty${latest_map_type}" local make_function="Make${latest_map_type}" # shellcheck disable=SC2016 @@ -45,6 +45,7 @@ function generate_latest_map() { cat <>"${out_file}" type ${entry_type} struct { + key string Timestamp time.Time ${json_timestamp} Value ${data_type} ${json_value} dummySelfer @@ -60,31 +61,53 @@ function generate_latest_map() { return e.Timestamp.Equal(e2.Timestamp) && e.Value == e2.Value } - // ${latest_map_type} holds latest ${data_type} instances. - type ${latest_map_type} struct { ps.Map } - - var ${empty_latest_map_variable} = ${latest_map_type}{ps.NewMap()} + // ${latest_map_type} holds latest ${data_type} instances, as a slice sorted by key. + type ${latest_map_type} struct { entries []${entry_type} } // ${make_function} makes an empty ${latest_map_type}. func ${make_function}() ${latest_map_type} { - return ${empty_latest_map_variable} + return ${latest_map_type}{} } // Size returns the number of elements. func (m ${latest_map_type}) Size() int { - if m.Map == nil { - return 0 - } - return m.Map.Size() + return len(m.entries) } // Merge produces a fresh ${latest_map_type} containing the keys from both inputs. // When both inputs contain the same key, the newer value is used. - func (m ${latest_map_type}) Merge(other ${latest_map_type}) ${latest_map_type} { - output := mergeMaps(m.Map, other.Map, func(a, b interface{}) bool { - return a.(*${entry_type}).Timestamp.Before(b.(*${entry_type}).Timestamp) - }) - return ${latest_map_type}{output} + func (m ${latest_map_type}) Merge(n ${latest_map_type}) ${latest_map_type} { + switch { + case m.entries == nil: + return n + case n.entries == nil: + return m + } + out := make([]${entry_type}, 0, len(m.entries)+len(n.entries)) + + i, j := 0, 0 + for i < len(m.entries) { + switch { + case j >= len(n.entries) || m.entries[i].key < n.entries[j].key: + out = append(out, m.entries[i]) + i++ + case m.entries[i].key == n.entries[j].key: + if m.entries[i].Timestamp.Before(n.entries[j].Timestamp) { + out = append(out, n.entries[j]) + } else { + out = append(out, m.entries[i]) + } + i++ + j++ + default: + out = append(out, n.entries[j]) + j++ + } + } + for ; j < len(n.entries); j++ { + out = append(out, n.entries[j]) + } + return ${latest_map_type}{out} } // Lookup the value for the given key. @@ -99,65 +122,135 @@ function generate_latest_map() { // LookupEntry returns the raw entry for the given key. func (m ${latest_map_type}) LookupEntry(key string) (${data_type}, time.Time, bool) { - if m.Map == nil { - var zero ${data_type} - return zero, time.Time{}, false + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + if i < len(m.entries) && m.entries[i].key == key { + return m.entries[i].Value, m.entries[i].Timestamp, true } - value, ok := m.Map.Lookup(key) - if !ok { - var zero ${data_type} - return zero, time.Time{}, false + var zero ${data_type} + return zero, time.Time{}, false + } + + // locate the position where key should go, and make room for it if not there already + func (m *${latest_map_type}) locate(key string) int { + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + // i is now the position where key should go, either at the end or in the middle + if i == len(m.entries) || m.entries[i].key != key { + m.entries = append(m.entries, ${entry_type}{}) + copy(m.entries[i+1:], m.entries[i:]) } - e := value.(*${entry_type}) - return e.Value, e.Timestamp, true + return i } // Set the value for the given key. func (m ${latest_map_type}) Set(key string, timestamp time.Time, value ${data_type}) ${latest_map_type} { - if m.Map == nil { - m.Map = ps.NewMap() + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + // i is now the position where key should go, either at the end or in the middle + oldEntries := m.entries + if i == len(m.entries) { + m.entries = make([]${entry_type}, len(oldEntries)+1) + copy(m.entries, oldEntries) + } else if m.entries[i].key == key { + m.entries = make([]${entry_type}, len(oldEntries)) + copy(m.entries, oldEntries) + } else { + m.entries = make([]${entry_type}, len(oldEntries)+1) + copy(m.entries, oldEntries[:i]) + copy(m.entries[i+1:], oldEntries[i:]) } - return ${latest_map_type}{m.Map.Set(key, &${entry_type}{Timestamp: timestamp, Value: value})} + m.entries[i] = ${entry_type}{key: key, Timestamp: timestamp, Value: value} + return m } // ForEach executes fn on each key value pair in the map. func (m ${latest_map_type}) ForEach(fn func(k string, timestamp time.Time, v ${data_type})) { - if m.Map != nil { - m.Map.ForEach(func(key string, value interface{}) { - fn(key, value.(*${entry_type}).Timestamp, value.(*${entry_type}).Value) - }) + for _, value := range m.entries { + fn(value.key, value.Timestamp, value.Value) } } // String returns the ${latest_map_type}'s string representation. func (m ${latest_map_type}) String() string { - return mapToString(m.Map) + buf := bytes.NewBufferString("{") + for _, val := range m.entries { + fmt.Fprintf(buf, "%s: %s,\n", val.key, val) + } + fmt.Fprintf(buf, "}") + return buf.String() } // DeepEqual tests equality with other ${latest_map_type}. func (m ${latest_map_type}) DeepEqual(n ${latest_map_type}) bool { - return mapEqual(m.Map, n.Map, func(val, otherValue interface{}) bool { - return val.(*${entry_type}).Equal(otherValue.(*${entry_type})) - }) + if m.Size() != n.Size() { + return false + } + for i := range m.entries { + if m.entries[i].key != n.entries[i].key || !m.entries[i].Equal(&n.entries[i]) { + return false + } + } + return true } // CodecEncodeSelf implements codec.Selfer. + // Duplicates the output for a built-in map without generating an + // intermediate copy of the data structure, to save time. Note this + // means we are using undocumented, internal APIs, which could break + // in the future. See https://github.com/weaveworks/scope/pull/1709 + // for more information. func (m *${latest_map_type}) CodecEncodeSelf(encoder *codec.Encoder) { - mapWrite(m.Map, encoder, func(encoder *codec.Encoder, val interface{}) { - val.(*${entry_type}).CodecEncodeSelf(encoder) - }) + z, r := codec.GenHelperEncoder(encoder) + if m.entries == nil { + r.EncodeNil() + return + } + r.EncodeMapStart(m.Size()) + for _, val := range m.entries { + z.EncSendContainerState(containerMapKey) + r.EncodeString(cUTF8, val.key) + z.EncSendContainerState(containerMapValue) + val.CodecEncodeSelf(encoder) + } + z.EncSendContainerState(containerMapEnd) } // CodecDecodeSelf implements codec.Selfer. + // Decodes the input as for a built-in map, without creating an + // intermediate copy of the data structure to save time. Uses + // undocumented, internal APIs as for CodecEncodeSelf. func (m *${latest_map_type}) CodecDecodeSelf(decoder *codec.Decoder) { - out := mapRead(decoder, func(isNil bool) interface{} { - value := &${entry_type}{} - if !isNil { - value.CodecDecodeSelf(decoder) - } - return value - }) - *m = ${latest_map_type}{out} + m.entries = nil + z, r := codec.GenHelperDecoder(decoder) + if r.TryDecodeAsNil() { + return + } + + length := r.ReadMapStart() + if length > 0 { + m.entries = make([]${entry_type}, 0, length) + } + for i := 0; length < 0 || i < length; i++ { + if length < 0 && r.CheckBreak() { + break + } + z.DecSendContainerState(containerMapKey) + var key string + if !r.TryDecodeAsNil() { + key = r.DecodeString() + } + i := m.locate(key) + m.entries[i].key = key + z.DecSendContainerState(containerMapValue) + if !r.TryDecodeAsNil() { + m.entries[i].CodecDecodeSelf(decoder) + } + } + z.DecSendContainerState(containerMapEnd) } // MarshalJSON shouldn't be used, use CodecEncodeSelf instead. diff --git a/report/latest_map_generated.go b/report/latest_map_generated.go index 570f6846c9..b3b78fffa1 100644 --- a/report/latest_map_generated.go +++ b/report/latest_map_generated.go @@ -4,14 +4,16 @@ package report import ( + "bytes" "fmt" + "sort" "time" "github.com/ugorji/go/codec" - "github.com/weaveworks/ps" ) type stringLatestEntry struct { + key string Timestamp time.Time `json:"timestamp"` Value string `json:"value"` dummySelfer @@ -27,31 +29,53 @@ func (e *stringLatestEntry) Equal(e2 *stringLatestEntry) bool { return e.Timestamp.Equal(e2.Timestamp) && e.Value == e2.Value } -// StringLatestMap holds latest string instances. -type StringLatestMap struct{ ps.Map } - -var emptyStringLatestMap = StringLatestMap{ps.NewMap()} +// StringLatestMap holds latest string instances, as a slice sorted by key. +type StringLatestMap struct{ entries []stringLatestEntry } // MakeStringLatestMap makes an empty StringLatestMap. func MakeStringLatestMap() StringLatestMap { - return emptyStringLatestMap + return StringLatestMap{} } // Size returns the number of elements. func (m StringLatestMap) Size() int { - if m.Map == nil { - return 0 - } - return m.Map.Size() + return len(m.entries) } // Merge produces a fresh StringLatestMap containing the keys from both inputs. // When both inputs contain the same key, the newer value is used. -func (m StringLatestMap) Merge(other StringLatestMap) StringLatestMap { - output := mergeMaps(m.Map, other.Map, func(a, b interface{}) bool { - return a.(*stringLatestEntry).Timestamp.Before(b.(*stringLatestEntry).Timestamp) - }) - return StringLatestMap{output} +func (m StringLatestMap) Merge(n StringLatestMap) StringLatestMap { + switch { + case m.entries == nil: + return n + case n.entries == nil: + return m + } + out := make([]stringLatestEntry, 0, len(m.entries)+len(n.entries)) + + i, j := 0, 0 + for i < len(m.entries) { + switch { + case j >= len(n.entries) || m.entries[i].key < n.entries[j].key: + out = append(out, m.entries[i]) + i++ + case m.entries[i].key == n.entries[j].key: + if m.entries[i].Timestamp.Before(n.entries[j].Timestamp) { + out = append(out, n.entries[j]) + } else { + out = append(out, m.entries[i]) + } + i++ + j++ + default: + out = append(out, n.entries[j]) + j++ + } + } + for ; j < len(n.entries); j++ { + out = append(out, n.entries[j]) + } + return StringLatestMap{out} } // Lookup the value for the given key. @@ -66,65 +90,135 @@ func (m StringLatestMap) Lookup(key string) (string, bool) { // LookupEntry returns the raw entry for the given key. func (m StringLatestMap) LookupEntry(key string) (string, time.Time, bool) { - if m.Map == nil { - var zero string - return zero, time.Time{}, false + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + if i < len(m.entries) && m.entries[i].key == key { + return m.entries[i].Value, m.entries[i].Timestamp, true } - value, ok := m.Map.Lookup(key) - if !ok { - var zero string - return zero, time.Time{}, false + var zero string + return zero, time.Time{}, false +} + +// locate the position where key should go, and make room for it if not there already +func (m *StringLatestMap) locate(key string) int { + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + // i is now the position where key should go, either at the end or in the middle + if i == len(m.entries) || m.entries[i].key != key { + m.entries = append(m.entries, stringLatestEntry{}) + copy(m.entries[i+1:], m.entries[i:]) } - e := value.(*stringLatestEntry) - return e.Value, e.Timestamp, true + return i } // Set the value for the given key. func (m StringLatestMap) Set(key string, timestamp time.Time, value string) StringLatestMap { - if m.Map == nil { - m.Map = ps.NewMap() + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + // i is now the position where key should go, either at the end or in the middle + oldEntries := m.entries + if i == len(m.entries) { + m.entries = make([]stringLatestEntry, len(oldEntries)+1) + copy(m.entries, oldEntries) + } else if m.entries[i].key == key { + m.entries = make([]stringLatestEntry, len(oldEntries)) + copy(m.entries, oldEntries) + } else { + m.entries = make([]stringLatestEntry, len(oldEntries)+1) + copy(m.entries, oldEntries[:i]) + copy(m.entries[i+1:], oldEntries[i:]) } - return StringLatestMap{m.Map.Set(key, &stringLatestEntry{Timestamp: timestamp, Value: value})} + m.entries[i] = stringLatestEntry{key: key, Timestamp: timestamp, Value: value} + return m } // ForEach executes fn on each key value pair in the map. func (m StringLatestMap) ForEach(fn func(k string, timestamp time.Time, v string)) { - if m.Map != nil { - m.Map.ForEach(func(key string, value interface{}) { - fn(key, value.(*stringLatestEntry).Timestamp, value.(*stringLatestEntry).Value) - }) + for _, value := range m.entries { + fn(value.key, value.Timestamp, value.Value) } } // String returns the StringLatestMap's string representation. func (m StringLatestMap) String() string { - return mapToString(m.Map) + buf := bytes.NewBufferString("{") + for _, val := range m.entries { + fmt.Fprintf(buf, "%s: %s,\n", val.key, val) + } + fmt.Fprintf(buf, "}") + return buf.String() } // DeepEqual tests equality with other StringLatestMap. func (m StringLatestMap) DeepEqual(n StringLatestMap) bool { - return mapEqual(m.Map, n.Map, func(val, otherValue interface{}) bool { - return val.(*stringLatestEntry).Equal(otherValue.(*stringLatestEntry)) - }) + if m.Size() != n.Size() { + return false + } + for i := range m.entries { + if m.entries[i].key != n.entries[i].key || !m.entries[i].Equal(&n.entries[i]) { + return false + } + } + return true } // CodecEncodeSelf implements codec.Selfer. +// Duplicates the output for a built-in map without generating an +// intermediate copy of the data structure, to save time. Note this +// means we are using undocumented, internal APIs, which could break +// in the future. See https://github.com/weaveworks/scope/pull/1709 +// for more information. func (m *StringLatestMap) CodecEncodeSelf(encoder *codec.Encoder) { - mapWrite(m.Map, encoder, func(encoder *codec.Encoder, val interface{}) { - val.(*stringLatestEntry).CodecEncodeSelf(encoder) - }) + z, r := codec.GenHelperEncoder(encoder) + if m.entries == nil { + r.EncodeNil() + return + } + r.EncodeMapStart(m.Size()) + for _, val := range m.entries { + z.EncSendContainerState(containerMapKey) + r.EncodeString(cUTF8, val.key) + z.EncSendContainerState(containerMapValue) + val.CodecEncodeSelf(encoder) + } + z.EncSendContainerState(containerMapEnd) } // CodecDecodeSelf implements codec.Selfer. +// Decodes the input as for a built-in map, without creating an +// intermediate copy of the data structure to save time. Uses +// undocumented, internal APIs as for CodecEncodeSelf. func (m *StringLatestMap) CodecDecodeSelf(decoder *codec.Decoder) { - out := mapRead(decoder, func(isNil bool) interface{} { - value := &stringLatestEntry{} - if !isNil { - value.CodecDecodeSelf(decoder) + m.entries = nil + z, r := codec.GenHelperDecoder(decoder) + if r.TryDecodeAsNil() { + return + } + + length := r.ReadMapStart() + if length > 0 { + m.entries = make([]stringLatestEntry, 0, length) + } + for i := 0; length < 0 || i < length; i++ { + if length < 0 && r.CheckBreak() { + break } - return value - }) - *m = StringLatestMap{out} + z.DecSendContainerState(containerMapKey) + var key string + if !r.TryDecodeAsNil() { + key = r.DecodeString() + } + i := m.locate(key) + m.entries[i].key = key + z.DecSendContainerState(containerMapValue) + if !r.TryDecodeAsNil() { + m.entries[i].CodecDecodeSelf(decoder) + } + } + z.DecSendContainerState(containerMapEnd) } // MarshalJSON shouldn't be used, use CodecEncodeSelf instead. @@ -138,6 +232,7 @@ func (*StringLatestMap) UnmarshalJSON(b []byte) error { } type nodeControlDataLatestEntry struct { + key string Timestamp time.Time `json:"timestamp"` Value NodeControlData `json:"value"` dummySelfer @@ -153,31 +248,53 @@ func (e *nodeControlDataLatestEntry) Equal(e2 *nodeControlDataLatestEntry) bool return e.Timestamp.Equal(e2.Timestamp) && e.Value == e2.Value } -// NodeControlDataLatestMap holds latest NodeControlData instances. -type NodeControlDataLatestMap struct{ ps.Map } - -var emptyNodeControlDataLatestMap = NodeControlDataLatestMap{ps.NewMap()} +// NodeControlDataLatestMap holds latest NodeControlData instances, as a slice sorted by key. +type NodeControlDataLatestMap struct{ entries []nodeControlDataLatestEntry } // MakeNodeControlDataLatestMap makes an empty NodeControlDataLatestMap. func MakeNodeControlDataLatestMap() NodeControlDataLatestMap { - return emptyNodeControlDataLatestMap + return NodeControlDataLatestMap{} } // Size returns the number of elements. func (m NodeControlDataLatestMap) Size() int { - if m.Map == nil { - return 0 - } - return m.Map.Size() + return len(m.entries) } // Merge produces a fresh NodeControlDataLatestMap containing the keys from both inputs. // When both inputs contain the same key, the newer value is used. -func (m NodeControlDataLatestMap) Merge(other NodeControlDataLatestMap) NodeControlDataLatestMap { - output := mergeMaps(m.Map, other.Map, func(a, b interface{}) bool { - return a.(*nodeControlDataLatestEntry).Timestamp.Before(b.(*nodeControlDataLatestEntry).Timestamp) - }) - return NodeControlDataLatestMap{output} +func (m NodeControlDataLatestMap) Merge(n NodeControlDataLatestMap) NodeControlDataLatestMap { + switch { + case m.entries == nil: + return n + case n.entries == nil: + return m + } + out := make([]nodeControlDataLatestEntry, 0, len(m.entries)+len(n.entries)) + + i, j := 0, 0 + for i < len(m.entries) { + switch { + case j >= len(n.entries) || m.entries[i].key < n.entries[j].key: + out = append(out, m.entries[i]) + i++ + case m.entries[i].key == n.entries[j].key: + if m.entries[i].Timestamp.Before(n.entries[j].Timestamp) { + out = append(out, n.entries[j]) + } else { + out = append(out, m.entries[i]) + } + i++ + j++ + default: + out = append(out, n.entries[j]) + j++ + } + } + for ; j < len(n.entries); j++ { + out = append(out, n.entries[j]) + } + return NodeControlDataLatestMap{out} } // Lookup the value for the given key. @@ -192,65 +309,135 @@ func (m NodeControlDataLatestMap) Lookup(key string) (NodeControlData, bool) { // LookupEntry returns the raw entry for the given key. func (m NodeControlDataLatestMap) LookupEntry(key string) (NodeControlData, time.Time, bool) { - if m.Map == nil { - var zero NodeControlData - return zero, time.Time{}, false + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + if i < len(m.entries) && m.entries[i].key == key { + return m.entries[i].Value, m.entries[i].Timestamp, true } - value, ok := m.Map.Lookup(key) - if !ok { - var zero NodeControlData - return zero, time.Time{}, false + var zero NodeControlData + return zero, time.Time{}, false +} + +// locate the position where key should go, and make room for it if not there already +func (m *NodeControlDataLatestMap) locate(key string) int { + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + // i is now the position where key should go, either at the end or in the middle + if i == len(m.entries) || m.entries[i].key != key { + m.entries = append(m.entries, nodeControlDataLatestEntry{}) + copy(m.entries[i+1:], m.entries[i:]) } - e := value.(*nodeControlDataLatestEntry) - return e.Value, e.Timestamp, true + return i } // Set the value for the given key. func (m NodeControlDataLatestMap) Set(key string, timestamp time.Time, value NodeControlData) NodeControlDataLatestMap { - if m.Map == nil { - m.Map = ps.NewMap() + i := sort.Search(len(m.entries), func(i int) bool { + return m.entries[i].key >= key + }) + // i is now the position where key should go, either at the end or in the middle + oldEntries := m.entries + if i == len(m.entries) { + m.entries = make([]nodeControlDataLatestEntry, len(oldEntries)+1) + copy(m.entries, oldEntries) + } else if m.entries[i].key == key { + m.entries = make([]nodeControlDataLatestEntry, len(oldEntries)) + copy(m.entries, oldEntries) + } else { + m.entries = make([]nodeControlDataLatestEntry, len(oldEntries)+1) + copy(m.entries, oldEntries[:i]) + copy(m.entries[i+1:], oldEntries[i:]) } - return NodeControlDataLatestMap{m.Map.Set(key, &nodeControlDataLatestEntry{Timestamp: timestamp, Value: value})} + m.entries[i] = nodeControlDataLatestEntry{key: key, Timestamp: timestamp, Value: value} + return m } // ForEach executes fn on each key value pair in the map. func (m NodeControlDataLatestMap) ForEach(fn func(k string, timestamp time.Time, v NodeControlData)) { - if m.Map != nil { - m.Map.ForEach(func(key string, value interface{}) { - fn(key, value.(*nodeControlDataLatestEntry).Timestamp, value.(*nodeControlDataLatestEntry).Value) - }) + for _, value := range m.entries { + fn(value.key, value.Timestamp, value.Value) } } // String returns the NodeControlDataLatestMap's string representation. func (m NodeControlDataLatestMap) String() string { - return mapToString(m.Map) + buf := bytes.NewBufferString("{") + for _, val := range m.entries { + fmt.Fprintf(buf, "%s: %s,\n", val.key, val) + } + fmt.Fprintf(buf, "}") + return buf.String() } // DeepEqual tests equality with other NodeControlDataLatestMap. func (m NodeControlDataLatestMap) DeepEqual(n NodeControlDataLatestMap) bool { - return mapEqual(m.Map, n.Map, func(val, otherValue interface{}) bool { - return val.(*nodeControlDataLatestEntry).Equal(otherValue.(*nodeControlDataLatestEntry)) - }) + if m.Size() != n.Size() { + return false + } + for i := range m.entries { + if m.entries[i].key != n.entries[i].key || !m.entries[i].Equal(&n.entries[i]) { + return false + } + } + return true } // CodecEncodeSelf implements codec.Selfer. +// Duplicates the output for a built-in map without generating an +// intermediate copy of the data structure, to save time. Note this +// means we are using undocumented, internal APIs, which could break +// in the future. See https://github.com/weaveworks/scope/pull/1709 +// for more information. func (m *NodeControlDataLatestMap) CodecEncodeSelf(encoder *codec.Encoder) { - mapWrite(m.Map, encoder, func(encoder *codec.Encoder, val interface{}) { - val.(*nodeControlDataLatestEntry).CodecEncodeSelf(encoder) - }) + z, r := codec.GenHelperEncoder(encoder) + if m.entries == nil { + r.EncodeNil() + return + } + r.EncodeMapStart(m.Size()) + for _, val := range m.entries { + z.EncSendContainerState(containerMapKey) + r.EncodeString(cUTF8, val.key) + z.EncSendContainerState(containerMapValue) + val.CodecEncodeSelf(encoder) + } + z.EncSendContainerState(containerMapEnd) } // CodecDecodeSelf implements codec.Selfer. +// Decodes the input as for a built-in map, without creating an +// intermediate copy of the data structure to save time. Uses +// undocumented, internal APIs as for CodecEncodeSelf. func (m *NodeControlDataLatestMap) CodecDecodeSelf(decoder *codec.Decoder) { - out := mapRead(decoder, func(isNil bool) interface{} { - value := &nodeControlDataLatestEntry{} - if !isNil { - value.CodecDecodeSelf(decoder) + m.entries = nil + z, r := codec.GenHelperDecoder(decoder) + if r.TryDecodeAsNil() { + return + } + + length := r.ReadMapStart() + if length > 0 { + m.entries = make([]nodeControlDataLatestEntry, 0, length) + } + for i := 0; length < 0 || i < length; i++ { + if length < 0 && r.CheckBreak() { + break } - return value - }) - *m = NodeControlDataLatestMap{out} + z.DecSendContainerState(containerMapKey) + var key string + if !r.TryDecodeAsNil() { + key = r.DecodeString() + } + i := m.locate(key) + m.entries[i].key = key + z.DecSendContainerState(containerMapValue) + if !r.TryDecodeAsNil() { + m.entries[i].CodecDecodeSelf(decoder) + } + } + z.DecSendContainerState(containerMapEnd) } // MarshalJSON shouldn't be used, use CodecEncodeSelf instead. diff --git a/report/latest_map_internal_test.go b/report/latest_map_internal_test.go index 7fe2ef1162..655fd32c05 100644 --- a/report/latest_map_internal_test.go +++ b/report/latest_map_internal_test.go @@ -72,7 +72,7 @@ func TestLatestMapDeepEquals(t *testing.T) { func nilStringLatestMap() StringLatestMap { m := MakeStringLatestMap() - m.Map = nil + m.entries = nil return m } From 736ae5e7c897599c95aa0df1b645554024651296 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 28 Sep 2017 08:45:54 +0000 Subject: [PATCH 2/4] Refactor: extract fn to make map for benchmarks --- report/latest_map_internal_test.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/report/latest_map_internal_test.go b/report/latest_map_internal_test.go index 655fd32c05..e66ac84123 100644 --- a/report/latest_map_internal_test.go +++ b/report/latest_map_internal_test.go @@ -126,21 +126,18 @@ func TestLatestMapMerge(t *testing.T) { } } -func BenchmarkLatestMapMerge(b *testing.B) { - var ( - left = MakeStringLatestMap() - right = MakeStringLatestMap() - now = time.Now() - ) - - // two large maps with some overlap - for i := 0; i < 1000; i++ { - left = left.Set(fmt.Sprint(i), now, "1") - } - for i := 700; i < 1700; i++ { - right = right.Set(fmt.Sprint(i), now.Add(1*time.Minute), "1") +func makeBenchmarkMap(start, finish int, timestamp time.Time) StringLatestMap { + ret := MakeStringLatestMap() + for i := start; i < finish; i++ { + ret = ret.Set(fmt.Sprint(i), timestamp, "1") } + return ret +} +func BenchmarkLatestMapMerge(b *testing.B) { + // two large maps with some overlap + left := makeBenchmarkMap(0, 1000, time.Now()) + right := makeBenchmarkMap(700, 1700, time.Now().Add(1*time.Minute)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { From a52c9df48cc436aba68ffe8e6ce99235bbee81b3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 28 Sep 2017 08:46:59 +0000 Subject: [PATCH 3/4] Added benchmarks for encode and decode of StringLatestMap --- report/latest_map_internal_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/report/latest_map_internal_test.go b/report/latest_map_internal_test.go index e66ac84123..ddb3c043b8 100644 --- a/report/latest_map_internal_test.go +++ b/report/latest_map_internal_test.go @@ -145,6 +145,28 @@ func BenchmarkLatestMapMerge(b *testing.B) { } } +func BenchmarkLatestMapEncode(b *testing.B) { + map1 := makeBenchmarkMap(0, 1000, time.Now()) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf := &bytes.Buffer{} + codec.NewEncoder(buf, &codec.MsgpackHandle{}).Encode(&map1) + } +} + +func BenchmarkLatestMapDecode(b *testing.B) { + map1 := makeBenchmarkMap(0, 1000, time.Now()) + buf := &bytes.Buffer{} + codec.NewEncoder(buf, &codec.MsgpackHandle{}).Encode(&map1) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var map1 StringLatestMap + codec.NewDecoderBytes(buf.Bytes(), &codec.MsgpackHandle{}).Decode(&map1) + } +} + func TestLatestMapEncoding(t *testing.T) { now := time.Now() want := MakeStringLatestMap(). From 5acece6e58452691826d2065ffcb503f4985e0a4 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 11 Oct 2017 09:50:06 +0000 Subject: [PATCH 4/4] Refactor: loop replaced with append() --- extras/generate_latest_map | 4 +--- report/latest_map_generated.go | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/extras/generate_latest_map b/extras/generate_latest_map index 2e7920cf13..183c35ac0b 100755 --- a/extras/generate_latest_map +++ b/extras/generate_latest_map @@ -104,9 +104,7 @@ function generate_latest_map() { j++ } } - for ; j < len(n.entries); j++ { - out = append(out, n.entries[j]) - } + out = append(out, n.entries[j:]...) return ${latest_map_type}{out} } diff --git a/report/latest_map_generated.go b/report/latest_map_generated.go index b3b78fffa1..52df7b113e 100644 --- a/report/latest_map_generated.go +++ b/report/latest_map_generated.go @@ -72,9 +72,7 @@ func (m StringLatestMap) Merge(n StringLatestMap) StringLatestMap { j++ } } - for ; j < len(n.entries); j++ { - out = append(out, n.entries[j]) - } + out = append(out, n.entries[j:]...) return StringLatestMap{out} } @@ -291,9 +289,7 @@ func (m NodeControlDataLatestMap) Merge(n NodeControlDataLatestMap) NodeControlD j++ } } - for ; j < len(n.entries); j++ { - out = append(out, n.entries[j]) - } + out = append(out, n.entries[j:]...) return NodeControlDataLatestMap{out} }