ThreadStats - Send correct metric type instead of always sending Gauge

datadog/threadstats/base.py

@@ -14,7 +14,6 @@
 
 # datadog
 from datadog.api.exceptions import ApiNotInitialized
-from datadog.threadstats.constants import MetricType
 from datadog.threadstats.events import EventsAggregator
 from datadog.threadstats.metrics import MetricsAggregator, Counter, Gauge, Histogram, Timing
 from datadog.threadstats.reporters import HttpReporter
@@ -211,8 +210,8 @@ def timing(self, metric_name, value, timestamp=None, tags=None, sample_rate=1, h
         >>> stats.timing("query.response.time", 1234)
         """
         if not self._disabled:
-            self._metric_aggregator.add_point(metric_name, tags, timestamp or time(), value, Timing,
-                                              sample_rate=sample_rate, host=host)
+            self._metric_aggregator.add_point(metric_name, tags, timestamp or time(), value,
+                                              Timing, sample_rate=sample_rate, host=host)
 
     @contextmanager
     def timer(self, metric_name, sample_rate=1, tags=None, host=None):
@@ -324,7 +323,7 @@ def _get_aggregate_metrics(self, flush_time=None):
 
         # FIXME: emit a dictionary from the aggregator
         metrics = []
-        for timestamp, value, name, tags, host in rolled_up_metrics:
+        for timestamp, value, name, tags, host, metric_type, interval in rolled_up_metrics:
             metric_tags = tags
             metric_name = name
 
@@ -342,10 +341,11 @@ def _get_aggregate_metrics(self, flush_time=None):
             metric = {
                 'metric': metric_name,
                 'points': [[timestamp, value]],
-                'type': MetricType.Gauge,
+                'type': metric_type,
                 'host': host,
                 'device': self.device,
-                'tags': metric_tags
+                'tags': metric_tags,
+                'interval': interval
             }
             metrics.append(metric)
         return metrics

datadog/threadstats/constants.py

@@ -2,6 +2,7 @@ class MetricType(object):
     Gauge = "gauge"
     Counter = "counter"
     Histogram = "histogram"
+    Rate = "rate"
 
 
 class MonitorType(object):

datadog/threadstats/metrics.py

@@ -6,6 +6,7 @@
 import itertools
 
 from datadog.util.compat import iternext
+from datadog.threadstats.constants import MetricType
 
 
 class Metric(object):
@@ -18,7 +19,7 @@ def add_point(self, value):
         """ Add a point to the given metric. """
         raise NotImplementedError()
 
-    def flush(self, timestamp):
+    def flush(self, timestamp, interval):
         """ Flush all metrics up to the given timestamp. """
         raise NotImplementedError()
 
@@ -37,12 +38,13 @@ def __init__(self, name, tags, host):
     def add_point(self, value):
         self.value = value
 
-    def flush(self, timestamp):
-        return [(timestamp, self.value, self.name, self.tags, self.host)]
+    def flush(self, timestamp, interval):
+        return [(timestamp, self.value, self.name, self.tags,
+                self.host, MetricType.Gauge, interval)]
 
 
 class Counter(Metric):
-    """ A counter metric. """
+    """ A metric that tracks a counter value. """
 
     stats_tag = 'c'
 
@@ -55,9 +57,10 @@ def __init__(self, name, tags, host):
     def add_point(self, value):
         self.count.append(value)
 
-    def flush(self, timestamp):
+    def flush(self, timestamp, interval):
         count = sum(self.count, 0)
-        return [(timestamp, count, self.name, self.tags, self.host)]
+        return [(timestamp, count/float(interval), self.name,
+                self.tags, self.host, MetricType.Rate, interval)]
 
 
 class Histogram(Metric):
@@ -88,21 +91,26 @@ def add_point(self, value):
             self.samples[random.randrange(0, self.sample_size)] = value
         self.count = iternext(self.iter_counter)
 
-    def flush(self, timestamp):
+    def flush(self, timestamp, interval):
         if not self.count:
             return []
         metrics = [
-            (timestamp, self.min, '%s.min' % self.name, self.tags, self.host),
-            (timestamp, self.max, '%s.max' % self.name, self.tags, self.host),
-            (timestamp, self.count, '%s.count' % self.name, self.tags, self.host),
-            (timestamp, self.average(), '%s.avg' % self.name, self.tags, self.host)
+            (timestamp, self.min, '%s.min' % self.name,
+             self.tags, self.host, MetricType.Gauge, interval),
+            (timestamp, self.max, '%s.max' % self.name,
+             self.tags, self.host, MetricType.Gauge, interval),
+            (timestamp, self.count/float(interval), '%s.count' % self.name,
+             self.tags, self.host, MetricType.Rate, interval),
+            (timestamp, self.average(), '%s.avg' % self.name,
+             self.tags, self.host, MetricType.Gauge, interval)
         ]
         length = len(self.samples)
         self.samples.sort()
         for p in self.percentiles:
             val = self.samples[int(round(p * length - 1))]
             name = '%s.%spercentile' % (self.name, int(p * 100))
-            metrics.append((timestamp, val, name, self.tags, self.host))
+            metrics.append((timestamp, val, name,
+                           self.tags, self.host, MetricType.Gauge, interval))
         return metrics
 
     def average(self):
@@ -147,5 +155,5 @@ def flush(self, timestamp):
         metrics = []
         for i in past_intervals:
             for m in list(self._metrics.pop(i).values()):
-                metrics += m.flush(i)
+                metrics += m.flush(i, self._roll_up_interval)
         return metrics

tests/unit/threadstats/test_threadstats.py

@@ -233,7 +233,7 @@ def test_histogram(self):
         nt.assert_equal(h1avg1['points'][0][1], 35)
         nt.assert_equal(h1cnt1['metric'], 'histogram.1.count')
         nt.assert_equal(h1cnt1['points'][0][0], 100.0)
-        nt.assert_equal(h1cnt1['points'][0][1], 4)
+        nt.assert_equal(h1cnt1['points'][0][1], 0.4)
         nt.assert_equal(h1min1['metric'], 'histogram.1.min')
         nt.assert_equal(h1min1['points'][0][1], 20)
         nt.assert_equal(h1max1['metric'], 'histogram.1.max')
@@ -248,7 +248,7 @@ def test_histogram(self):
         nt.assert_equal(h1avg2['points'][0][1], 40)
         nt.assert_equal(h1cnt2['metric'], 'histogram.1.count')
         nt.assert_equal(h1cnt2['points'][0][0], 110.0)
-        nt.assert_equal(h1cnt2['points'][0][1], 3)
+        nt.assert_equal(h1cnt2['points'][0][1], 0.3)
         nt.assert_equal(h1752['metric'], 'histogram.1.75percentile')
         nt.assert_equal(h1752['points'][0][0], 110.0)
         nt.assert_equal(h1752['points'][0][1], 40.0)
@@ -261,7 +261,7 @@ def test_histogram(self):
         nt.assert_equal(h2avg1['points'][0][1], 40)
         nt.assert_equal(h2cnt1['metric'], 'histogram.2.count')
         nt.assert_equal(h2cnt1['points'][0][0], 100.0)
-        nt.assert_equal(h2cnt1['points'][0][1], 1)
+        nt.assert_equal(h2cnt1['points'][0][1], 0.1)
 
         # Flush again ensure they're gone.
         dog.reporter.metrics = []
@@ -347,7 +347,7 @@ def test_counter(self):
         (first, second) = metrics
         nt.assert_equal(first['metric'], 'test.counter.1')
         nt.assert_equal(first['points'][0][0], 1000.0)
-        nt.assert_equal(first['points'][0][1], 3)
+        nt.assert_equal(first['points'][0][1], 0.3)
         nt.assert_equal(second['metric'], 'test.counter.2')
 
         # Test decrement
@@ -361,7 +361,7 @@ def test_counter(self):
         first, = metrics
         nt.assert_equal(first['metric'], 'test.counter.1')
         nt.assert_equal(first['points'][0][0], 1000.0)
-        nt.assert_equal(first['points'][0][1], 8)
+        nt.assert_equal(first['points'][0][1], 0.8)
         nt.assert_equal(second['metric'], 'test.counter.2')
 
         # Flush again and make sure we're progressing.
@@ -416,11 +416,11 @@ def test_tags(self):
         [c1, c2, c3, g1, g2, g3] = metrics
         (nt.assert_equal(c['metric'], 'counter') for c in [c1, c2, c3])
         nt.assert_equal(c1['tags'], None)
-        nt.assert_equal(c1['points'][0][1], 1)
+        nt.assert_equal(c1['points'][0][1], 0.1)
         nt.assert_equal(c2['tags'], ['env:production', 'db'])
-        nt.assert_equal(c2['points'][0][1], 1)
+        nt.assert_equal(c2['points'][0][1], 0.1)
         nt.assert_equal(c3['tags'], ['env:staging'])
-        nt.assert_equal(c3['points'][0][1], 1)
+        nt.assert_equal(c3['points'][0][1], 0.1)
 
         (nt.assert_equal(c['metric'], 'gauge') for c in [g1, g2, g3])
         nt.assert_equal(g1['tags'], None)
@@ -454,12 +454,14 @@ def test_constant_tags(self):
 
         # Assertions on gauges
         self.assertMetric(name='gauge', value=10, tags=["type:constant"], count=1)
-        self.assertMetric(name="gauge", value=15, tags=["env:production", "db", "type:constant"], count=1)  # noqa
+        self.assertMetric(name="gauge", value=15, 
+                          tags=["env:production", "db", "type:constant"], count=1)  # noqa
         self.assertMetric(name="gauge", value=20, tags=["env:staging", "type:constant"], count=1)
 
         # Assertions on counters
         self.assertMetric(name="counter", value=1, tags=["type:constant"], count=1)
-        self.assertMetric(name="counter", value=1, tags=["env:production", "db", "type:constant"], count=1)  # noqa
+        self.assertMetric(name="counter", value=1, 
+                          tags=["env:production", "db", "type:constant"], count=1)  # noqa
         self.assertMetric(name="counter", value=1, tags=["env:staging", "type:constant"], count=1)
 
         # Ensure histograms work as well.
@@ -527,13 +529,13 @@ def test_host(self):
         (nt.assert_equal(c['metric'], 'counter') for c in [c1, c2, c3])
         nt.assert_equal(c1['host'], None)
         nt.assert_equal(c1['tags'], None)
-        nt.assert_equal(c1['points'][0][1], 2)
+        nt.assert_equal(c1['points'][0][1], 0.2)
         nt.assert_equal(c2['host'], 'test')
         nt.assert_equal(c2['tags'], None)
-        nt.assert_equal(c2['points'][0][1], 1)
+        nt.assert_equal(c2['points'][0][1], 0.1)
         nt.assert_equal(c3['host'], 'test')
         nt.assert_equal(c3['tags'], ['tag'])
-        nt.assert_equal(c3['points'][0][1], 2)
+        nt.assert_equal(c3['points'][0][1], 0.2)
 
         (nt.assert_equal(g['metric'], 'gauge') for g in [g1, g2, g3])
         nt.assert_equal(g1['host'], None)
@@ -671,3 +673,33 @@ def test_tags_from_environment_and_constant(self):
         nt.assert_equal(event['date_happened'], event1_date_happened)
         nt.assert_equal(event['tags'], [event1_tag] + constant_tags + test_tags)
         dog.start(flush_interval=1, roll_up_interval=1)
+
+    def test_metric_type(self):
+        """
+        Checks the submitted metric's metric type.
+        """
+        # Set up ThreadStats with a namespace
+        dog = ThreadStats(namespace="foo")
+        dog.start(roll_up_interval=1, flush_in_thread=False)
+        reporter = dog.reporter = self.reporter
+
+        # Send a few metrics
+        dog.gauge("gauge", 20, timestamp=100.0)
+        dog.increment("counter", timestamp=100.0)
+        dog.histogram('histogram.1', 20, 100.0)
+        dog.flush(200.0)
+
+        (first, second, p75, p85, p95, p99, avg, cnt, 
+        max_, min_) = self.sort_metrics(reporter.metrics)
+
+		# Assert Metric type
+        nt.assert_equal(first['type'], 'rate')
+        nt.assert_equal(second['type'], 'gauge')
+        nt.assert_equal(p75['type'], 'gauge')
+        nt.assert_equal(p85['type'], 'gauge')
+        nt.assert_equal(p95['type'], 'gauge')
+        nt.assert_equal(p99['type'], 'gauge')
+        nt.assert_equal(avg['type'], 'gauge')
+        nt.assert_equal(cnt['type'], 'rate')
+        nt.assert_equal(max_['type'], 'gauge')
+        nt.assert_equal(min_['type'], 'gauge')