-
Notifications
You must be signed in to change notification settings - Fork 4.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Move to DogStatsD. #10238
Move to DogStatsD. #10238
Changes from 7 commits
961b6e1
e186660
83c9cbd
0eb580e
d2bf1c7
c37f2e0
42bec12
beea7ac
6501a08
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,128 @@ | ||
/* | ||
* Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
*/ | ||
|
||
package io.airbyte.metrics; | ||
|
||
import com.timgroup.statsd.NonBlockingStatsDClientBuilder; | ||
import com.timgroup.statsd.StatsDClient; | ||
import java.util.Map; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.slf4j.MDC; | ||
|
||
/** | ||
* Light wrapper around the DogsStatsD client to make using the client slightly more ergonomic. | ||
* <p> | ||
* This class mainly exists to help Airbyte instrument/debug application on Airbyte Cloud. | ||
* <p> | ||
* Open source users are free to turn this on and consume the same metrics. | ||
*/ | ||
@Slf4j | ||
public class DogstatsdMetricSingleton { | ||
|
||
private static DogstatsdMetricSingleton instance; | ||
private final StatsDClient statsDClient; | ||
private final boolean instancePublish; | ||
|
||
public DogstatsdMetricSingleton(final String appName, final boolean publish) { | ||
instancePublish = publish; | ||
statsDClient = new NonBlockingStatsDClientBuilder() | ||
.prefix(appName) | ||
.hostname(System.getenv("DD_AGENT_HOST")) | ||
.port(Integer.parseInt(System.getenv("DD_DOGSTATSD_PORT"))) | ||
.build(); | ||
} | ||
|
||
public static synchronized DogstatsdMetricSingleton getInstance() { | ||
if (instance == null) { | ||
throw new RuntimeException("You must initialize configuration with the initializeMonitoringServiceDaemon() method before getting an instance."); | ||
} | ||
return instance; | ||
} | ||
|
||
public synchronized static void initialize(final String appName, final Map<String, String> mdc, final boolean publish) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why is the MDC important here? Shouldn't the initialization be run from some thread that has the MDC configured? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good point - I think it's left over from when I was testing. You should be right that we don't need this anymore. Will remove. |
||
if (instance != null) { | ||
throw new RuntimeException("You cannot initialize configuration more than once."); | ||
} | ||
if (publish) { | ||
MDC.setContextMap(mdc); | ||
log.info("Starting DogStatsD client.."); | ||
// The second constructor argument ('true') makes this server start as a separate daemon thread. | ||
// http://prometheus.github.io/client_java/io/prometheus/client/exporter/HTTPServer.html#HTTPServer-int-boolean- | ||
instance = new DogstatsdMetricSingleton(appName, publish); | ||
} | ||
} | ||
|
||
/** | ||
* Increment or decrement a counter. | ||
* | ||
* @param name of counter. | ||
* @param amt to adjust. | ||
* @param tags | ||
*/ | ||
public void count(final String name, final double amt, final String... tags) { | ||
if (instancePublish) { | ||
log.info("publishing count, name: {}, value: {}", name, amt); | ||
statsDClient.count(name, amt, tags); | ||
} | ||
} | ||
|
||
/** | ||
* Record the latest value for a gauge. | ||
* | ||
* @param name of gauge. | ||
* @param val to record. | ||
* @param tags | ||
*/ | ||
public void gauge(final String name, final double val, final String... tags) { | ||
if (instancePublish) { | ||
log.info("publishing gauge, name: {}, value: {}", name, val); | ||
statsDClient.gauge(name, val, tags); | ||
} | ||
} | ||
|
||
/** | ||
* Submit a single execution time aggregated locally by the Agent. Use this if approximate stats are | ||
* sufficient. | ||
* | ||
* @param name of histogram. | ||
* @param val of time to record. | ||
* @param tags | ||
*/ | ||
public void recordTimeLocal(final String name, final double val, final String... tags) { | ||
if (instancePublish) { | ||
log.info("recording histogram, name: {}, value: {}", name, val); | ||
statsDClient.histogram(name, val, tags); | ||
} | ||
} | ||
|
||
/** | ||
* Submit a single execution time aggregated globally by Datadog. Use this for precise stats. | ||
* | ||
* @param name of distribution. | ||
* @param val of time to record. | ||
* @param tags | ||
*/ | ||
public void recordTimeGlobal(final String name, final double val, final String... tags) { | ||
if (instancePublish) { | ||
log.info("recording distribution, name: {}, value: {}", name, val); | ||
statsDClient.distribution(name, val, tags); | ||
} | ||
} | ||
|
||
/** | ||
* Wrapper of {@link #recordTimeGlobal(String, double, String...)} with a runnable for convenience. | ||
* | ||
* @param name | ||
* @param runnable | ||
* @param tags | ||
*/ | ||
public void recordTimeGlobal(final String name, final Runnable runnable, final String... tags) { | ||
final long start = System.currentTimeMillis(); | ||
runnable.run(); | ||
final long end = System.currentTimeMillis(); | ||
final long val = end - start; | ||
recordTimeGlobal(name, val, tags); | ||
} | ||
|
||
} |
This file was deleted.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should this say
.. with the initialize() method
?Also, what is the downside of combining
getInstance()
andinitialize()
into the same method? The behavior would be that ifinstance == null
, it initializes and returns. Ifinstance != null
, it just returns. That approach feels like it would be a bit more ergonomic, no?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ah yes good catch on the error message.
this is a small thing - the main downside would be the
initialise
method has more parameters than thegetInstance
method. we'd have to combine those parameters if we combined the methods. because of this I prefer to keep these separate for now.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@davinchia I'm curious how you are thinking about consumers of this class. If I want to use the DogstatsdMetricSingleton, how do I know if it has been initialized or not? Is it expected that a consumer would call
getInstance
wrapped with a try-catch to catch a RuntimeException in the case that it was already initialized, and callinitialize
in that case? That doesn't feel very ergonomic to me, but let me know if I'm thinking about this wrongThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good question - I'm trying to follow the general singleton pattern where
initialize
is called in the main method, andgetInstance
is called everywhere else. Although it's possible to catch, most folks don't really catch the RTE since it's usually understood the singleton is initialised when the JVM first starts up.I agree with you having to do
getInstance
is somewhat annoying, but this is a fairly common pattern and I don't really think it's worth trying to get rid of.I can't see how to not have a simply method signature if we combine
initialize
andgetInstance
and I didn't want to spend more time playing with this.Happy to discuss more/review any ideas/approaches you come up with!