Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Consolidate honeycomb metrics to use single lock & fix concurrent read/write #511

Merged
merged 7 commits into from
Sep 14, 2022
Merged

Consolidate honeycomb metrics to use single lock & fix concurrent read/write #511

Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7461fd8
consolidate honeycomb metrics to use single lock
MikeGoldsmith Sep 12, 2022
e723614
use explict lock / unlock when collecting metrics
MikeGoldsmith Sep 12, 2022
f4241ef
switch to using RWLock
MikeGoldsmith Sep 13, 2022
a63997b
introduce getOrAdd generic func
MikeGoldsmith Sep 13, 2022
6443267
Update metrics/honeycomb.go
MikeGoldsmith Sep 13, 2022
b96bbfb
update createGauge typo
MikeGoldsmith Sep 14, 2022
68985ca
fix typos / docs
MikeGoldsmith Sep 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions metrics/honeycomb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,10 @@ type HoneycombMetrics struct {
UpstreamTransport *http.Transport `inject:"upstreamTransport"`
Version string `inject:"version"`

countersLock sync.Mutex
counters map[string]*counter
gaugesLock sync.Mutex
gauges map[string]*gauge
histogramsLock sync.Mutex
histograms map[string]*histogram
lock sync.Mutex
MikeGoldsmith marked this conversation as resolved.
Show resolved Hide resolved
counters map[string]*counter
gauges map[string]*gauge
histograms map[string]*histogram

libhClient *libhoney.Client

Expand Down Expand Up @@ -228,26 +226,23 @@ func (h *HoneycombMetrics) reportToHoneycommb(ctx context.Context) {
"api_host": ev.APIHost,
"dataset": ev.Dataset,
}
h.countersLock.Lock()
MikeGoldsmith marked this conversation as resolved.
Show resolved Hide resolved

h.lock.Lock()
for _, count := range h.counters {
count.lock.Lock()
ev.AddField(PrefixMetricName(h.prefix, count.name), count.val)
count.val = 0
count.lock.Unlock()
}
h.countersLock.Unlock()

h.gaugesLock.Lock()
for _, gauge := range h.gauges {
gauge.lock.Lock()
ev.AddField(PrefixMetricName(h.prefix, gauge.name), gauge.val)
// gauges should remain where they are until changed
// gauge.val = 0
gauge.lock.Unlock()
}
h.gaugesLock.Unlock()

h.histogramsLock.Lock()
for _, histogram := range h.histograms {
histogram.lock.Lock()
if len(histogram.vals) != 0 {
Expand All @@ -265,7 +260,7 @@ func (h *HoneycombMetrics) reportToHoneycommb(ctx context.Context) {
}
histogram.lock.Unlock()
}
h.histogramsLock.Unlock()
h.lock.Unlock()

ev.Send()
}
Expand All @@ -281,10 +276,11 @@ func average(vals []float64) float64 {
}

func (h *HoneycombMetrics) Register(name string, metricType string) {
h.lock.Lock()
MikeGoldsmith marked this conversation as resolved.
Show resolved Hide resolved
defer h.lock.Unlock()

switch metricType {
case "counter":
h.countersLock.Lock()
defer h.countersLock.Unlock()
// inside the lock, let's not race to create the counter
_, ok := h.counters[name]
if !ok {
Expand All @@ -294,8 +290,6 @@ func (h *HoneycombMetrics) Register(name string, metricType string) {
h.counters[name] = newCounter
}
case "gauge":
h.gaugesLock.Lock()
defer h.gaugesLock.Unlock()
_, ok := h.gauges[name]
if !ok {
newGauge := &gauge{
Expand All @@ -304,8 +298,6 @@ func (h *HoneycombMetrics) Register(name string, metricType string) {
h.gauges[name] = newGauge
}
case "histogram":
h.histogramsLock.Lock()
defer h.histogramsLock.Unlock()
_, ok := h.histograms[name]
if !ok {
newGauge := &histogram{
Expand All @@ -320,6 +312,9 @@ func (h *HoneycombMetrics) Register(name string, metricType string) {
}

func (h *HoneycombMetrics) Count(name string, n interface{}) {
h.lock.Lock()
MikeGoldsmith marked this conversation as resolved.
Show resolved Hide resolved
defer h.lock.Unlock()

count, ok := h.counters[name]
if !ok {
h.Register(name, "counter")
Expand All @@ -335,6 +330,9 @@ func (h *HoneycombMetrics) Increment(name string) {
}

func (h *HoneycombMetrics) Gauge(name string, val interface{}) {
h.lock.Lock()
defer h.lock.Unlock()

gauge, ok := h.gauges[name]
if !ok {
h.Register(name, "gauge")
Expand All @@ -346,6 +344,9 @@ func (h *HoneycombMetrics) Gauge(name string, val interface{}) {
}

func (h *HoneycombMetrics) Histogram(name string, obs interface{}) {
h.lock.Lock()
defer h.lock.Unlock()

histogram, ok := h.histograms[name]
if !ok {
h.Register(name, "histogram")
Expand Down