src, test: node internals' postmortem metadata

Before these changes, only V8 added postmortem metadata to Node's
binary, limiting the possibilities for debugger's developers to add some
features that rely on investigating Node's internal structures.

These changes are first steps towards empowering debug tools to
navigate Node's internal structures. One example of what can be
achieved with this is shown at nodejs/llnode#122 (a command which prints
information about handles and requests on the queue for a core dump
file). Node postmortem metadata are prefixed with nodedbg_.

This also adds tests to validate if all postmortem metadata are
calculated correctly, plus some documentation on what is postmortem
metadata and a few care to be taken to avoid breaking it.

Ref: nodejs/llnode#122
Ref: nodejs/post-mortem#46

Backport-PR-URL: #19176
PR-URL: #14901
Refs: nodejs/post-mortem#46
Reviewed-By: Michael Dawson <michael_dawson@ca.ibm.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

doc/guides/node-postmortem-support.md

@@ -0,0 +1,72 @@
+# Postmortem Support
+
+Postmortem metadata are constants present in the final build which can be used
+by debuggers and other tools to navigate through internal structures of software
+when analyzing its memory (either on a running process or a core dump). Node
+provides this metadata in its builds for V8 and Node internal structures.
+
+
+### V8 Postmortem metadata
+
+V8 prefixes all postmortem constants with `v8dbg_`, and they allow inspection of
+objects on the heap as well as object properties and references. V8 generates
+those symbols with a script (`deps/v8/tools/gen-postmortem-metadata.py`), and
+Node always includes these constants in the final build.
+
+### Node Debug Symbols
+
+Node prefixes all postmortem constants with `nodedbg_`, and they complement V8
+constants by providing ways to inspect Node-specific structures, like
+`node::Environment`, `node::BaseObject` and its descendants, classes from
+`src/utils.h` and others. Those constants are declared in
+`src/node_postmortem_metadata.cc`, and most of them are calculated at compile
+time.
+
+#### Calculating offset of class members
+
+Node constants referring to the offset of class members in memory are calculated
+at compile time. Because of that, those class members must be at a fixed offset
+from the start of the class. That's not a problem in most cases, but it also
+means that those members should always come after any templated member on the
+class definition.
+
+For example, if we want to add a constant with the offset for
+`ReqWrap::req_wrap_queue_`, it should be defined after `ReqWrap::req_`, because
+`sizeof(req_)` depends on the type of T, which means the class definition should
+be like this:
+
+```c++
+template <typename T>
+class ReqWrap : public AsyncWrap {
+ private:
+  // req_wrap_queue_ comes before any templated member, which places it in a
+  // fixed offset from the start of the class
+  ListNode<ReqWrap> req_wrap_queue_;
+
+  T req_;
+};
+```
+
+instead of:
+
+```c++
+template <typename T>
+class ReqWrap : public AsyncWrap {
+ private:
+  T req_;
+
+  // req_wrap_queue_ comes after a templated member, which means it won't be in
+  // a fixed offset from the start of the class
+  ListNode<ReqWrap> req_wrap_queue_;
+};
+```
+
+There are also tests on `test/cctest/test_node_postmortem_metadata.cc` to make
+sure all Node postmortem metadata are calculated correctly.
+
+## Tools and References
+
+* [llnode](https://github.com/nodejs/llnode): LLDB plugin
+* [`mdb_v8`](https://github.com/joyent/mdb_v8): mdb plugin
+* [nodejs/post-mortem](https://github.com/nodejs/post-mortem): Node.js
+post-mortem working group

node.gyp

@@ -288,6 +288,7 @@
         'src/node_os.cc',
         'src/node_platform.cc',
         'src/node_perf.cc',
+        'src/node_postmortem_metadata.cc',
         'src/node_serdes.cc',
         'src/node_trace_events.cc',
         'src/node_url.cc',
@@ -944,6 +945,7 @@
         'test/cctest/node_test_fixture.cc',
         'test/cctest/test_aliased_buffer.cc',
         'test/cctest/test_base64.cc',
+        'test/cctest/test_node_postmortem_metadata.cc',
         'test/cctest/test_environment.cc',
         'test/cctest/test_util.cc',
         'test/cctest/test_url.cc'
@@ -972,6 +974,30 @@
       ],
 
       'conditions': [
+        ['node_target_type!="static_library"', {
+          'libraries': [
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)async_wrap.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)handle_wrap.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)env.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_buffer.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_debug_options.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_i18n.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_perf.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_platform.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_url.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)util.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)string_bytes.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)string_search.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)stream_base.<(OBJ_SUFFIX)',
+            '<(OBJ_PATH)<(OBJ_SEPARATOR)node_constants.<(OBJ_SUFFIX)',
+            '<(OBJ_TRACING_PATH)<(OBJ_SEPARATOR)agent.<(OBJ_SUFFIX)',
+            '<(OBJ_TRACING_PATH)<(OBJ_SEPARATOR)node_trace_buffer.<(OBJ_SUFFIX)',
+            '<(OBJ_TRACING_PATH)<(OBJ_SEPARATOR)node_trace_writer.<(OBJ_SUFFIX)',
+            '<(OBJ_TRACING_PATH)<(OBJ_SEPARATOR)trace_event.<(OBJ_SUFFIX)',
+            '<(OBJ_GEN_PATH)<(OBJ_SEPARATOR)node_javascript.<(OBJ_SUFFIX)',
+          ],
+        }],
         [ 'node_use_openssl=="true"', {
           'conditions': [
             ['node_target_type!="static_library"', {

src/base-object.h

@@ -65,6 +65,11 @@ class BaseObject {
   static inline void WeakCallback(
       const v8::WeakCallbackInfo<Type>& data);
 
+  // persistent_handle_ needs to be at a fixed offset from the start of the
+  // class because it is used by src/node_postmortem_metadata.cc to calculate
+  // offsets and generate debug symbols for BaseObject, which assumes that the
+  // position of members in memory are predictable. For more information please
+  // refer to `doc/guides/node-postmortem-support.md`
   v8::Persistent<v8::Object> persistent_handle_;
   Environment* env_;
 };

src/env.h

@@ -732,6 +732,12 @@ class Environment {
   std::unique_ptr<inspector::Agent> inspector_agent_;
 #endif
 
+  // handle_wrap_queue_ and req_wrap_queue_ needs to be at a fixed offset from
+  // the start of the class because it is used by
+  // src/node_postmortem_metadata.cc to calculate offsets and generate debug
+  // symbols for Environment, which assumes that the position of members in
+  // memory are predictable. For more information please refer to
+  // `doc/guides/node-postmortem-support.md`
   HandleWrapQueue handle_wrap_queue_;
   ReqWrapQueue req_wrap_queue_;
   ListHead<HandleCleanup,

src/handle_wrap.h

@@ -81,6 +81,11 @@ class HandleWrap : public AsyncWrap {
   friend class Environment;
   friend void GetActiveHandles(const v8::FunctionCallbackInfo<v8::Value>&);
   static void OnClose(uv_handle_t* handle);
+  // handle_wrap_queue_ needs to be at a fixed offset from the start of the
+  // class because it is used by src/node_postmortem_metadata.cc to calculate
+  // offsets and generate debug symbols for HandleWrap, which assumes that the
+  // position of members in memory are predictable. For more information please
+  // refer to `doc/guides/node-postmortem-support.md`
   ListNode<HandleWrap> handle_wrap_queue_;
   enum { kInitialized, kClosing, kClosingWithCallback, kClosed } state_;
   uv_handle_t* const handle_;

src/node_postmortem_metadata.cc

@@ -0,0 +1,118 @@
+// Need to import standard headers before redefining private, otherwise it
+// won't compile.
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <bitset>
+#include <cctype>
+#include <climits>
+#include <cmath>
+#include <cstdarg>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <deque>
+#include <exception>
+#include <forward_list>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iosfwd>
+#include <iostream>
+#include <istream>
+#include <iterator>
+#include <limits>
+#include <list>
+#include <map>
+#include <memory>
+#include <new>
+#include <ostream>
+#include <queue>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <streambuf>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <typeinfo>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace node {
+// Forward declaration needed before redefining private.
+int GenDebugSymbols();
+}  // namespace node
+
+
+#define private friend int GenDebugSymbols(); private
+
+#include "env.h"
+#include "base-object-inl.h"
+#include "handle_wrap.h"
+#include "util-inl.h"
+#include "req-wrap.h"
+#include "v8abbr.h"
+
+#define NODEDBG_SYMBOL(Name)  nodedbg_ ## Name
+
+// nodedbg_offset_CLASS__MEMBER__TYPE: Describes the offset to a class member.
+#define NODEDBG_OFFSET(Class, Member, Type) \
+    NODEDBG_SYMBOL(offset_ ## Class ## __ ## Member ## __ ## Type)
+
+// These are the constants describing Node internal structures. Every constant
+// should use the format described above.  These constants are declared as
+// global integers so that they'll be present in the generated node binary. They
+// also need to be declared outside any namespace to avoid C++ name-mangling.
+#define NODE_OFFSET_POSTMORTEM_METADATA(V)                                    \
+  V(BaseObject, persistent_handle_, v8_Persistent_v8_Object,                  \
+    BaseObject::persistent_handle_)                                           \
+  V(Environment, handle_wrap_queue_, Environment_HandleWrapQueue,             \
+    Environment::handle_wrap_queue_)                                          \
+  V(Environment, req_wrap_queue_, Environment_ReqWrapQueue,                   \
+    Environment::req_wrap_queue_)                                             \
+  V(HandleWrap, handle_wrap_queue_, ListNode_HandleWrap,                      \
+    HandleWrap::handle_wrap_queue_)                                           \
+  V(Environment_HandleWrapQueue, head_, ListNode_HandleWrap,                  \
+    Environment::HandleWrapQueue::head_)                                      \
+  V(ListNode_HandleWrap, next_, uintptr_t, ListNode<HandleWrap>::next_)       \
+  V(ReqWrap, req_wrap_queue_, ListNode_ReqWrapQueue,                          \
+    ReqWrap<uv_req_t>::req_wrap_queue_)                                       \
+  V(Environment_ReqWrapQueue, head_, ListNode_ReqWrapQueue,                   \
+    Environment::ReqWrapQueue::head_)                                         \
+  V(ListNode_ReqWrap, next_, uintptr_t, ListNode<ReqWrap<uv_req_t>>::next_)
+
+extern "C" {
+int nodedbg_const_Environment__kContextEmbedderDataIndex__int;
+uintptr_t nodedbg_offset_ExternalString__data__uintptr_t;
+
+#define V(Class, Member, Type, Accessor)                                      \
+  NODE_EXTERN uintptr_t NODEDBG_OFFSET(Class, Member, Type);
+  NODE_OFFSET_POSTMORTEM_METADATA(V)
+#undef V
+}
+
+namespace node {
+
+int GenDebugSymbols() {
+  nodedbg_const_Environment__kContextEmbedderDataIndex__int =
+      Environment::kContextEmbedderDataIndex;
+
+  nodedbg_offset_ExternalString__data__uintptr_t = NODE_OFF_EXTSTR_DATA;
+
+  #define V(Class, Member, Type, Accessor)                                    \
+    NODEDBG_OFFSET(Class, Member, Type) = OffsetOf(&Accessor);
+    NODE_OFFSET_POSTMORTEM_METADATA(V)
+  #undef V
+
+  return 1;
+}
+
+int debug_symbols_generated = GenDebugSymbols();
+
+}  // namespace node

src/req-wrap.h

@@ -27,9 +27,13 @@ class ReqWrap : public AsyncWrap {
  protected:
   // req_wrap_queue_ needs to be at a fixed offset from the start of the class
   // because it is used by ContainerOf to calculate the address of the embedding
-  // ReqWrap. ContainerOf compiles down to simple, fixed pointer arithmetic.
-  // sizeof(req_) depends on the type of T, so req_wrap_queue_ would
-  // no longer be at a fixed offset if it came after req_.
+  // ReqWrap. ContainerOf compiles down to simple, fixed pointer arithmetic. It
+  // is also used by src/node_postmortem_metadata.cc to calculate offsets and
+  // generate debug symbols for ReqWrap, which assumes that the position of
+  // members in memory are predictable. sizeof(req_) depends on the type of T,
+  // so req_wrap_queue_ would no longer be at a fixed offset if it came after
+  // req_. For more information please refer to
+  // `doc/guides/node-postmortem-support.md`
   T req_;
 };
 

src/util-inl.h

@@ -154,13 +154,17 @@ typename ListHead<T, M>::Iterator ListHead<T, M>::end() const {
   return Iterator(const_cast<ListNode<T>*>(&head_));
 }
 
+template <typename Inner, typename Outer>
+constexpr uintptr_t OffsetOf(Inner Outer::*field) {
+  return reinterpret_cast<uintptr_t>(&(static_cast<Outer*>(0)->*field));
+}
+
 template <typename Inner, typename Outer>
 ContainerOfHelper<Inner, Outer>::ContainerOfHelper(Inner Outer::*field,
                                                    Inner* pointer)
-    : pointer_(reinterpret_cast<Outer*>(
-          reinterpret_cast<uintptr_t>(pointer) -
-          reinterpret_cast<uintptr_t>(&(static_cast<Outer*>(0)->*field)))) {
-}
+    : pointer_(
+        reinterpret_cast<Outer*>(
+            reinterpret_cast<uintptr_t>(pointer) - OffsetOf(field))) {}
 
 template <typename Inner, typename Outer>
 template <typename TypeName>

test/cctest/node_test_fixture.h

@@ -5,6 +5,7 @@
 #include "gtest/gtest.h"
 #include "node.h"
 #include "node_platform.h"
+#include "node_internals.h"
 #include "env.h"
 #include "v8.h"
 #include "libplatform/libplatform.h"
@@ -69,6 +70,13 @@ class NodeTestFixture : public ::testing::Test {
     v8::Isolate::CreateParams params_;
     params_.array_buffer_allocator = allocator_.get();
     isolate_ = v8::Isolate::New(params_);
+
+    // As the TracingController is stored globally, we only need to create it
+    // one time for all tests.
+    if (node::tracing::TraceEventHelper::GetTracingController() == nullptr) {
+      node::tracing::TraceEventHelper::SetTracingController(
+          new v8::TracingController());
+    }
   }
 
   virtual void TearDown() {
@@ -88,4 +96,48 @@ class NodeTestFixture : public ::testing::Test {
       v8::ArrayBuffer::Allocator::NewDefaultAllocator()};
 };
 
+
+class EnvironmentTestFixture : public NodeTestFixture {
+ public:
+  class Env {
+   public:
+    Env(const v8::HandleScope& handle_scope, const Argv& argv) {
+      auto isolate = handle_scope.GetIsolate();
+      context_ = v8::Context::New(isolate);
+      CHECK(!context_.IsEmpty());
+      context_->Enter();
+
+      isolate_data_ = node::CreateIsolateData(isolate,
+                                              NodeTestFixture::CurrentLoop());
+      CHECK_NE(nullptr, isolate_data_);
+      environment_ = node::CreateEnvironment(isolate_data_,
+                                             context_,
+                                             1, *argv,
+                                             argv.nr_args(), *argv);
+      CHECK_NE(nullptr, environment_);
+    }
+
+    ~Env() {
+      environment_->CleanupHandles();
+      node::FreeEnvironment(environment_);
+      node::FreeIsolateData(isolate_data_);
+      context_->Exit();
+    }
+
+    node::Environment* operator*() const {
+      return environment_;
+    }
+
+    v8::Local<v8::Context> context()  const {
+      return context_;
+    }
+
+   private:
+    v8::Local<v8::Context> context_;
+    node::IsolateData* isolate_data_;
+    node::Environment* environment_;
+    DISALLOW_COPY_AND_ASSIGN(Env);
+  };
+};
+
 #endif  // TEST_CCTEST_NODE_TEST_FIXTURE_H_