add runtime metrics

.circleci/config.yml

@@ -22,6 +22,9 @@ node-core-base: &node-core-base
         paths:
           - ./node_modules
           - ./yarn.lock
+    - run:
+        name: Compile native code
+        command: yarn rebuild
     - run:
         name: Unit tests
         command: yarn test:core

.gitignore

@@ -91,10 +91,6 @@ typings/
 
 ### VisualStudioCode ###
 .vscode/*
-!.vscode/settings.json
-!.vscode/tasks.json
-!.vscode/launch.json
-!.vscode/extensions.json
 .history
 
 
@@ -104,5 +100,6 @@ package-lock.json
 yarn.lock
 out
 versions
+build
 docs/test.js
 !test/node_modules

LICENSE-3rdparty.csv

@@ -1,6 +1,7 @@
 Component,Origin,License,Copyright
 require,@types/node,MIT,Copyright Authors
 require,async-hook-jl,MIT,Copyright 2015 Andreas Madsen
+require,hot-shots,MIT,Copyright 2011 Steve Ivy
 require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,lodash.kebabcase,MIT,Copyright JS Foundation and other contributors
@@ -12,6 +13,7 @@ require,lodash.uniq,MIT,Copyright JS Foundation and other contributors
 require,methods,MIT,Copyright 2013-2014 TJ Holowaychuk 2013-2014 TJ Holowaychuk
 require,module-details-from-path,MIT,Copyright 2016 Thomas Watson Steen
 require,msgpack-lite,MIT,Copyright 2015 Yusuke Kawasaki
+require,nan,MIT,Copyright 2018 NAN contributors
 require,opentracing,MIT,Copyright 2016 Resonance Labs Inc
 require,parent-module,MIT,Copyright Sindre Sorhus
 require,path-to-regexp,MIT,Copyright 2014 Blake Embrey

azure-pipelines.yml

@@ -2,8 +2,14 @@ pool:
   vmImage: vs2017-win2016
 
 steps:
-- script: yarn versions && yarn install
+- bash: yarn versions
+  displayName: Versions
+
+- bash: yarn install
   displayName: Install dependencies
 
+- bash: yarn rebuild
+  displayName: Compile native code
+
 - bash: yarn test:core
   displayName: Test Core

benchmark/core.js

@@ -18,7 +18,7 @@ const Writer = proxyquire('../src/writer', {
 const Sampler = require('../src/sampler')
 const format = require('../src/format')
 const encode = require('../src/encode')
-const config = new Config({ service: 'benchmark' })
+const config = new Config('benchmark', '', { service: 'benchmark' })
 
 const suite = benchmark('core')
 

binding.gyp

@@ -0,0 +1,19 @@
+{
+  "targets": [{
+    "target_name": "metrics",
+    "sources": [
+      "src/native/metrics/Collector.cpp",
+      "src/native/metrics/EventLoop.cpp",
+      "src/native/metrics/GarbageCollection.cpp",
+      "src/native/metrics/Heap.cpp",
+      "src/native/metrics/Histogram.cpp",
+      "src/native/metrics/Object.cpp",
+      "src/native/metrics/Process.cpp",
+      "src/native/metrics/main.cpp"
+    ],
+    "include_dirs": [
+      "src",
+      "<!(node -e \"require('nan')\")"
+    ]
+  }]
+}

docs/test.ts

@@ -16,7 +16,9 @@ tracer.init({
   enabled: true,
   analytics: true,
   env: 'test',
-  experimental: true,
+  experimental: {
+    runtimeMetrics: true
+  },
   hostname: 'agent',
   logger: {
     error (message: string) {},

index.d.ts

@@ -210,7 +210,13 @@ export declare interface TracerOptions {
    * Experimental features can be enabled all at once by using true or individually using key / value pairs.
    * @default {}
    */
-  experimental?: {} | boolean;
+  experimental?: {
+    /**
+     * Whether to enable runtime metrics.
+     * @default false
+     */
+    runtimeMetrics?: boolean
+  } | boolean;
 
   /**
    * Whether to load all built-in plugins.

package.json

@@ -5,6 +5,8 @@
   "main": "index.js",
   "typings": "index.d.ts",
   "scripts": {
+    "rebuild": "node-gyp rebuild",
+    "install": "node scripts/post_install.js",
     "bench": "node benchmark",
     "type:doc": "cd docs && yarn && yarn build",
     "type:test": "cd docs && yarn && yarn test",
@@ -40,6 +42,7 @@
   "dependencies": {
     "@types/node": "^10.12.18",
     "async-hook-jl": "^1.7.6",
+    "hot-shots": "^5.0.0",
     "int64-buffer": "^0.1.9",
     "koalas": "^1.0.2",
     "lodash.kebabcase": "^4.1.1",
@@ -51,6 +54,7 @@
     "methods": "^1.1.2",
     "module-details-from-path": "^1.0.3",
     "msgpack-lite": "^0.1.26",
+    "nan": "^2.12.1",
     "opentracing": ">=0.12.1",
     "parent-module": "^0.1.0",
     "path-to-regexp": "^0.1.2",

scripts/post_install.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const execSync = require('child_process').execSync
+
+if (process.env.DD_NATIVE_METRICS === 'true') {
+  try {
+    execSync('npm run rebuild --scripts-prepend-node-path', { stdio: [0, 1, 2] })
+  } catch (e) {
+    /* eslint-disable no-console */
+    console.log()
+    console.log([
+      'Compilation of native modules failed.',
+      'Falling back to JavaScript only.',
+      'Some functionalities may not be available.'
+    ].join(' '))
+    console.log()
+    /* eslint-enable no-console */
+  }
+}

src/config.js

@@ -5,8 +5,8 @@ const platform = require('./platform')
 const coalesce = require('koalas')
 
 class Config {
-  constructor (service, options) {
-    options = typeof service === 'object' ? service : options || {}
+  constructor (service, runtimeId, options) {
+    options = options || {}
 
     const enabled = coalesce(options.enabled, platform.env('DD_TRACE_ENABLED'), true)
     const debug = coalesce(options.debug, platform.env('DD_TRACE_DEBUG'), false)
@@ -17,8 +17,7 @@ class Config {
     const hostname = coalesce(
       options.hostname,
       platform.env('DD_AGENT_HOST'),
-      platform.env('DD_TRACE_AGENT_HOSTNAME'),
-      'localhost'
+      platform.env('DD_TRACE_AGENT_HOSTNAME')
     )
     const port = coalesce(options.port, platform.env('DD_TRACE_AGENT_PORT'), 8126)
     const sampleRate = coalesce(Math.min(Math.max(options.sampleRate, 0), 1), 1)
@@ -30,16 +29,25 @@ class Config {
     this.debug = String(debug) === 'true'
     this.logInjection = String(logInjection) === 'true'
     this.env = env
-    this.url = url ? new URL(url) : new URL(`${protocol}://${hostname}:${port}`)
-    this.tags = Object.assign({}, options.tags)
+    this.url = url ? new URL(url) : new URL(`${protocol}://${hostname || 'localhost'}:${port}`)
+    this.hostname = hostname || this.url.hostname
     this.flushInterval = flushInterval
     this.bufferSize = 100000
     this.sampleRate = sampleRate
     this.logger = options.logger
     this.plugins = !!plugins
     this.service = coalesce(options.service, platform.env('DD_SERVICE_NAME'), service, 'node')
     this.analytics = String(analytics) === 'true'
+    this.runtimeId = coalesce(runtimeId, '')
+    this.tags = Object.assign({ 'runtime-id': this.runtimeId }, options.tags)
+    this.experimental = {
+      runtimeMetrics: isFlagEnabled(options.experimental, 'runtimeMetrics')
+    }
   }
 }
 
+function isFlagEnabled (obj, prop) {
+  return obj === true || (typeof obj === 'object' && obj !== null && obj[prop])
+}
+
 module.exports = Config

src/native/metrics/Collector.cpp

@@ -0,0 +1,7 @@
+#include "Collector.hpp"
+
+namespace datadog {
+  uint64_t Collector::time_to_micro(uv_timeval_t timeval) {
+    return timeval.tv_sec * 1e6 + timeval.tv_usec;
+  }
+}

src/native/metrics/Collector.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <stdint.h>
+#include <uv.h>
+
+#include "Object.hpp"
+
+namespace datadog {
+  class Collector {
+    public:
+      virtual void inject(Object carrier) = 0;
+    protected:
+      virtual uint64_t time_to_micro(uv_timeval_t timeval);
+  };
+}

src/native/metrics/EventLoop.cpp

@@ -0,0 +1,49 @@
+#include "EventLoop.hpp"
+
+namespace datadog {
+  // http://docs.libuv.org/en/v1.x/design.html#the-i-o-loop
+  EventLoop::EventLoop() {
+    uv_check_init(uv_default_loop(), &check_handle_);
+    uv_unref(reinterpret_cast<uv_handle_t*>(&check_handle_));
+
+    check_handle_.data = (void*)this;
+    check_usage_ = usage();
+  }
+
+  EventLoop::~EventLoop() {
+    uv_check_stop(&check_handle_);
+  }
+
+  void EventLoop::check_cb (uv_check_t* handle) {
+    EventLoop* self = (EventLoop*)handle->data;
+    self->tick();
+  }
+
+  void EventLoop::tick () {
+    uint64_t usage = this->usage();
+
+    histogram_.add(usage - check_usage_);
+    check_usage_ = usage;
+  }
+
+  void EventLoop::enable() {
+    uv_check_start(&check_handle_, &EventLoop::check_cb);
+  }
+
+  void EventLoop::disable() {
+    uv_check_stop(&check_handle_);
+    histogram_.reset();
+  }
+
+  uint64_t EventLoop::usage() {
+    uv_rusage_t usage;
+    uv_getrusage(&usage);
+
+    return time_to_micro(usage.ru_utime) + time_to_micro(usage.ru_stime);
+  }
+
+  void EventLoop::inject(Object carrier) {
+    carrier.set("eventLoop", histogram_);
+    histogram_.reset();
+  }
+}

src/native/metrics/EventLoop.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <stdint.h>
+#include <uv.h>
+
+#include "Collector.hpp"
+#include "Histogram.hpp"
+
+namespace datadog {
+  class EventLoop : public Collector {
+    public:
+      EventLoop();
+      ~EventLoop();
+
+      void enable();
+      void disable();
+      void tick();
+      void inject(Object carrier);
+    protected:
+      static void check_cb (uv_check_t* handle);
+    private:
+      uv_check_t check_handle_;
+      uint64_t check_usage_;
+      Histogram histogram_;
+
+      uint64_t usage();
+  };
+}

src/native/metrics/GarbageCollection.cpp

@@ -0,0 +1,33 @@
+#include "GarbageCollection.hpp"
+
+namespace datadog {
+  GarbageCollection::GarbageCollection() {
+    pause_[v8::GCType::kGCTypeAll] = Histogram();
+  }
+
+  void GarbageCollection::before(v8::GCType type) {
+    start_time_ = uv_hrtime();
+  }
+
+  void GarbageCollection::after(v8::GCType type) {
+    uint64_t usage = uv_hrtime() - start_time_;
+
+    if (pause_.find(type) == pause_.end()) {
+      pause_[type] = Histogram();
+    }
+
+    pause_[type].add(usage);
+    pause_[v8::GCType::kGCTypeAll].add(usage);
+  }
+
+  void GarbageCollection::inject(Object carrier) {
+    Object value;
+
+    for (std::map<v8::GCType, Histogram>::iterator it = pause_.begin(); it != pause_.end(); ++it) {
+      value.set(types_[it->first], it->second);
+      it->second.reset();
+    }
+
+    carrier.set("gc", value);
+  }
+}

src/native/metrics/GarbageCollection.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <map>
+#include <string>
+#include <stdint.h>
+#include <uv.h>
+#include <v8.h>
+
+#include "Collector.hpp"
+#include "Histogram.hpp"
+#include "Object.hpp"
+
+namespace datadog {
+  class GarbageCollection : public Collector {
+    public:
+      GarbageCollection();
+
+      void before(v8::GCType type);
+      void after(v8::GCType type);
+      void inject(Object carrier);
+    private:
+      std::map<v8::GCType, Histogram> pause_;
+      std::map<unsigned char, std::string> types_ = {
+        { 1, "scavenge" },
+        { 2, "mark_sweep_compact" },
+        { 3, "all" }, // Node 4
+        { 4, "incremental_marking" },
+        { 8, "process_weak_callbacks" },
+        { 15, "all" }
+      };
+      uint64_t start_time_;
+  };
+}