Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[RFC] report: encode timestamps in a compact form #2440

Closed
wants to merge 1 commit into from
Closed

[RFC] report: encode timestamps in a compact form #2440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 33 additions & 14 deletions extras/generate_latest_map
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ function generate_header() {
package report

import (
"encoding/base64"
"fmt"
"time"

Expand All @@ -39,25 +40,29 @@ function generate_latest_map() {
local make_function="Make${latest_map_type}"

# shellcheck disable=SC2016
local json_timestamp='`json:"timestamp"`'
local json_timestamp='`json:"timestamp,omitempty"`'
# shellcheck disable=SC2016
local json_small_timestamp='`json:"t,omitempty"`'
# shellcheck disable=SC2016
local json_value='`json:"value"`'

cat <<EOF >>"${out_file}"
type ${entry_type} struct {
Timestamp time.Time ${json_timestamp}
Timestamp *time.Time ${json_timestamp}
SmallTimestamp *string ${json_small_timestamp}

Value ${data_type} ${json_value}
dummySelfer
}

// String returns the StringLatestEntry's string representation.
func (e *${entry_type}) String() string {
return fmt.Sprintf("%v (%s)", e.Value, e.Timestamp.String())
return fmt.Sprintf("%v (%s)", e.Value, *e.SmallTimestamp)
}

// Equal returns true if the supplied StringLatestEntry is equal to this one.
func (e *${entry_type}) Equal(e2 *${entry_type}) bool {
return e.Timestamp.Equal(e2.Timestamp) && e.Value == e2.Value
return *e.SmallTimestamp == *e2.SmallTimestamp && e.Value == e2.Value
}

// ${latest_map_type} holds latest ${data_type} instances.
Expand Down Expand Up @@ -88,7 +93,7 @@ function generate_latest_map() {
// When both inputs contain the same key, the newer value is used.
func (m ${latest_map_type}) Merge(other ${latest_map_type}) ${latest_map_type} {
output := mergeMaps(m.Map, other.Map, func(a, b interface{}) bool {
return a.(*${entry_type}).Timestamp.Before(b.(*${entry_type}).Timestamp)
return a.(*${entry_type}).Timestamp.Before(*b.(*${entry_type}).Timestamp)
})
return ${latest_map_type}{output}
}
Expand All @@ -115,15 +120,17 @@ function generate_latest_map() {
return zero, time.Time{}, false
}
e := value.(*${entry_type})
return e.Value, e.Timestamp, true
return e.Value, *e.Timestamp, true
}

// Set the value for the given key.
func (m ${latest_map_type}) Set(key string, timestamp time.Time, value ${data_type}) ${latest_map_type} {
if m.Map == nil {
m.Map = ps.NewMap()
}
return ${latest_map_type}{m.Map.Set(key, &${entry_type}{Timestamp: timestamp, Value: value})}
bytesBuf, _ := timestamp.UTC().MarshalBinary()
smallTimestamp := base64.StdEncoding.EncodeToString(bytesBuf)
return ${latest_map_type}{m.Map.Set(key, &${entry_type}{Timestamp: &timestamp, SmallTimestamp: &smallTimestamp, Value: value})}
}

// Delete the value for the given key.
Expand All @@ -137,9 +144,9 @@ function generate_latest_map() {
// ForEach executes fn on each key value pair in the map.
func (m ${latest_map_type}) ForEach(fn func(k string, timestamp time.Time, v ${data_type})) {
if m.Map != nil {
m.Map.ForEach(func(key string, value interface{}) {
fn(key, value.(*${entry_type}).Timestamp, value.(*${entry_type}).Value)
})
m.Map.ForEach(func(key string, value interface{}) {
fn(key, *value.(*${entry_type}).Timestamp, value.(*${entry_type}).Value)
})
}
}

Expand All @@ -151,16 +158,18 @@ function generate_latest_map() {
// DeepEqual tests equality with other ${latest_map_type}.
func (m ${latest_map_type}) DeepEqual(n ${latest_map_type}) bool {
return mapEqual(m.Map, n.Map, func(val, otherValue interface{}) bool {
return val.(*${entry_type}).Equal(otherValue.(*${entry_type}))
return val.(*${entry_type}).Equal(otherValue.(*${entry_type}))
})
}

func (m ${latest_map_type}) toIntermediate() map[string]${entry_type} {
intermediate := make(map[string]${entry_type}, m.Size())
if m.Map != nil {
m.Map.ForEach(func(key string, val interface{}) {
intermediate[key] = *val.(*${entry_type})
})
m.Map.ForEach(func(key string, val interface{}) {
var tmp = *val.(*${entry_type})
tmp.Timestamp = nil
intermediate[key] = tmp
})
}
return intermediate
}
Expand All @@ -180,6 +189,16 @@ function generate_latest_map() {
value := &${entry_type}{}
if !isNil {
value.CodecDecodeSelf(decoder)
if value.SmallTimestamp == nil {
defaultSmallTimestamp := ""
value.SmallTimestamp = &defaultSmallTimestamp
}
if value.Timestamp == nil {
decoded, _ := base64.StdEncoding.DecodeString(*value.SmallTimestamp)
ts := time.Time{}
ts.UnmarshalBinary(decoded)
value.Timestamp = &ts
}
}
return value
})
Expand Down
69 changes: 51 additions & 18 deletions report/latest_map_generated.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package report

import (
"encoding/base64"
"fmt"
"time"

Expand All @@ -12,19 +13,21 @@ import (
)

type stringLatestEntry struct {
Timestamp time.Time `json:"timestamp"`
Value string `json:"value"`
Timestamp *time.Time `json:"timestamp,omitempty"`
SmallTimestamp *string `json:"t,omitempty"`

Value string `json:"value"`
dummySelfer
}

// String returns the StringLatestEntry's string representation.
func (e *stringLatestEntry) String() string {
return fmt.Sprintf("%v (%s)", e.Value, e.Timestamp.String())
return fmt.Sprintf("%v (%s)", e.Value, *e.SmallTimestamp)
}

// Equal returns true if the supplied StringLatestEntry is equal to this one.
func (e *stringLatestEntry) Equal(e2 *stringLatestEntry) bool {
return e.Timestamp.Equal(e2.Timestamp) && e.Value == e2.Value
return *e.SmallTimestamp == *e2.SmallTimestamp && e.Value == e2.Value
}

// StringLatestMap holds latest string instances.
Expand Down Expand Up @@ -55,7 +58,7 @@ func (m StringLatestMap) Size() int {
// When both inputs contain the same key, the newer value is used.
func (m StringLatestMap) Merge(other StringLatestMap) StringLatestMap {
output := mergeMaps(m.Map, other.Map, func(a, b interface{}) bool {
return a.(*stringLatestEntry).Timestamp.Before(b.(*stringLatestEntry).Timestamp)
return a.(*stringLatestEntry).Timestamp.Before(*b.(*stringLatestEntry).Timestamp)
})
return StringLatestMap{output}
}
Expand All @@ -82,15 +85,17 @@ func (m StringLatestMap) LookupEntry(key string) (string, time.Time, bool) {
return zero, time.Time{}, false
}
e := value.(*stringLatestEntry)
return e.Value, e.Timestamp, true
return e.Value, *e.Timestamp, true
}

// Set the value for the given key.
func (m StringLatestMap) Set(key string, timestamp time.Time, value string) StringLatestMap {
if m.Map == nil {
m.Map = ps.NewMap()
}
return StringLatestMap{m.Map.Set(key, &stringLatestEntry{Timestamp: timestamp, Value: value})}
bytesBuf, _ := timestamp.UTC().MarshalBinary()
smallTimestamp := base64.StdEncoding.EncodeToString(bytesBuf)
return StringLatestMap{m.Map.Set(key, &stringLatestEntry{Timestamp: &timestamp, SmallTimestamp: &smallTimestamp, Value: value})}
}

// Delete the value for the given key.
Expand All @@ -105,7 +110,7 @@ func (m StringLatestMap) Delete(key string) StringLatestMap {
func (m StringLatestMap) ForEach(fn func(k string, timestamp time.Time, v string)) {
if m.Map != nil {
m.Map.ForEach(func(key string, value interface{}) {
fn(key, value.(*stringLatestEntry).Timestamp, value.(*stringLatestEntry).Value)
fn(key, *value.(*stringLatestEntry).Timestamp, value.(*stringLatestEntry).Value)
})
}
}
Expand All @@ -126,7 +131,9 @@ func (m StringLatestMap) toIntermediate() map[string]stringLatestEntry {
intermediate := make(map[string]stringLatestEntry, m.Size())
if m.Map != nil {
m.Map.ForEach(func(key string, val interface{}) {
intermediate[key] = *val.(*stringLatestEntry)
var tmp = *val.(*stringLatestEntry)
tmp.Timestamp = nil
intermediate[key] = tmp
})
}
return intermediate
Expand All @@ -147,6 +154,16 @@ func (m *StringLatestMap) CodecDecodeSelf(decoder *codec.Decoder) {
value := &stringLatestEntry{}
if !isNil {
value.CodecDecodeSelf(decoder)
if value.SmallTimestamp == nil {
defaultSmallTimestamp := ""
value.SmallTimestamp = &defaultSmallTimestamp
}
if value.Timestamp == nil {
decoded, _ := base64.StdEncoding.DecodeString(*value.SmallTimestamp)
ts := time.Time{}
ts.UnmarshalBinary(decoded)
value.Timestamp = &ts
}
}
return value
})
Expand All @@ -164,19 +181,21 @@ func (*StringLatestMap) UnmarshalJSON(b []byte) error {
}

type nodeControlDataLatestEntry struct {
Timestamp time.Time `json:"timestamp"`
Value NodeControlData `json:"value"`
Timestamp *time.Time `json:"timestamp,omitempty"`
SmallTimestamp *string `json:"t,omitempty"`

Value NodeControlData `json:"value"`
dummySelfer
}

// String returns the StringLatestEntry's string representation.
func (e *nodeControlDataLatestEntry) String() string {
return fmt.Sprintf("%v (%s)", e.Value, e.Timestamp.String())
return fmt.Sprintf("%v (%s)", e.Value, *e.SmallTimestamp)
}

// Equal returns true if the supplied StringLatestEntry is equal to this one.
func (e *nodeControlDataLatestEntry) Equal(e2 *nodeControlDataLatestEntry) bool {
return e.Timestamp.Equal(e2.Timestamp) && e.Value == e2.Value
return *e.SmallTimestamp == *e2.SmallTimestamp && e.Value == e2.Value
}

// NodeControlDataLatestMap holds latest NodeControlData instances.
Expand Down Expand Up @@ -207,7 +226,7 @@ func (m NodeControlDataLatestMap) Size() int {
// When both inputs contain the same key, the newer value is used.
func (m NodeControlDataLatestMap) Merge(other NodeControlDataLatestMap) NodeControlDataLatestMap {
output := mergeMaps(m.Map, other.Map, func(a, b interface{}) bool {
return a.(*nodeControlDataLatestEntry).Timestamp.Before(b.(*nodeControlDataLatestEntry).Timestamp)
return a.(*nodeControlDataLatestEntry).Timestamp.Before(*b.(*nodeControlDataLatestEntry).Timestamp)
})
return NodeControlDataLatestMap{output}
}
Expand All @@ -234,15 +253,17 @@ func (m NodeControlDataLatestMap) LookupEntry(key string) (NodeControlData, time
return zero, time.Time{}, false
}
e := value.(*nodeControlDataLatestEntry)
return e.Value, e.Timestamp, true
return e.Value, *e.Timestamp, true
}

// Set the value for the given key.
func (m NodeControlDataLatestMap) Set(key string, timestamp time.Time, value NodeControlData) NodeControlDataLatestMap {
if m.Map == nil {
m.Map = ps.NewMap()
}
return NodeControlDataLatestMap{m.Map.Set(key, &nodeControlDataLatestEntry{Timestamp: timestamp, Value: value})}
bytesBuf, _ := timestamp.UTC().MarshalBinary()
smallTimestamp := base64.StdEncoding.EncodeToString(bytesBuf)
return NodeControlDataLatestMap{m.Map.Set(key, &nodeControlDataLatestEntry{Timestamp: &timestamp, SmallTimestamp: &smallTimestamp, Value: value})}
}

// Delete the value for the given key.
Expand All @@ -257,7 +278,7 @@ func (m NodeControlDataLatestMap) Delete(key string) NodeControlDataLatestMap {
func (m NodeControlDataLatestMap) ForEach(fn func(k string, timestamp time.Time, v NodeControlData)) {
if m.Map != nil {
m.Map.ForEach(func(key string, value interface{}) {
fn(key, value.(*nodeControlDataLatestEntry).Timestamp, value.(*nodeControlDataLatestEntry).Value)
fn(key, *value.(*nodeControlDataLatestEntry).Timestamp, value.(*nodeControlDataLatestEntry).Value)
})
}
}
Expand All @@ -278,7 +299,9 @@ func (m NodeControlDataLatestMap) toIntermediate() map[string]nodeControlDataLat
intermediate := make(map[string]nodeControlDataLatestEntry, m.Size())
if m.Map != nil {
m.Map.ForEach(func(key string, val interface{}) {
intermediate[key] = *val.(*nodeControlDataLatestEntry)
var tmp = *val.(*nodeControlDataLatestEntry)
tmp.Timestamp = nil
intermediate[key] = tmp
})
}
return intermediate
Expand All @@ -299,6 +322,16 @@ func (m *NodeControlDataLatestMap) CodecDecodeSelf(decoder *codec.Decoder) {
value := &nodeControlDataLatestEntry{}
if !isNil {
value.CodecDecodeSelf(decoder)
if value.SmallTimestamp == nil {
defaultSmallTimestamp := ""
value.SmallTimestamp = &defaultSmallTimestamp
}
if value.Timestamp == nil {
decoded, _ := base64.StdEncoding.DecodeString(*value.SmallTimestamp)
ts := time.Time{}
ts.UnmarshalBinary(decoded)
value.Timestamp = &ts
}
}
return value
})
Expand Down
40 changes: 39 additions & 1 deletion report/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package report

import (
"encoding/base64"
"math"
"time"

Expand Down Expand Up @@ -51,12 +52,49 @@ type Metric struct {
type Sample struct {
Timestamp time.Time `json:"date"`
Value float64 `json:"value"`
dummySelfer
}

// CodecEncodeSelf implements codec.Selfer.
func (s *Sample) CodecEncodeSelf(encoder *codec.Encoder) {
if s == nil {
encoder.Encode(nil)
return
}
type sampleMarshalled struct {
Timestamp string `json:"t"`
Value float64 `json:"v"`
}
encoder.Encode(func(s *Sample) sampleMarshalled {
bytesBuf, _ := s.Timestamp.UTC().MarshalBinary()
smallTimestamp := base64.StdEncoding.EncodeToString(bytesBuf)
return sampleMarshalled{smallTimestamp, s.Value}
}(s))
}

// CodecDecodeSelf implements codec.Selfer.
func (s *Sample) CodecDecodeSelf(decoder *codec.Decoder) {
type sampleMarshalled struct {
Timestamp string `json:"t"`
Value float64 `json:"v"`
}

value := &sampleMarshalled{}
err := decoder.Decode(value)
if err != nil {
s = nil
return
}
decoded, _ := base64.StdEncoding.DecodeString(value.Timestamp)
ts := time.Time{}
ts.UnmarshalBinary(decoded)
*s = Sample{Timestamp: ts, Value: value.Value, dummySelfer: struct{}{}}
}

// MakeSingletonMetric makes a metric with a single value
func MakeSingletonMetric(t time.Time, v float64) Metric {
return Metric{
Samples: []Sample{{t, v}},
Samples: []Sample{{t, v, struct{}{}}},
Min: v,
Max: v,
First: t,
Expand Down