Print stack traces before committing suicide

Summary:
Functions my_pstack() and macro abort_with_stack_traces(),
assert_with_stack_traces, and assert_0_with_stack_traces are introduced
in this diff. When they get called, the stack traces of all threads will
be print out into error log before abort or assert. Many abort(),
assert(0), and assert(expr) in the code have been replaced with these
macros, this will help for the investigations of the fatal failures. The
asserts in ndb, debug mode, plugins, or with complex logic were not
replaced.

Test Plan: Jenkins 'arc unit' all passed.

Reviewers: steaphan

Reviewed By: steaphan

include/my_stacktrace.h

@@ -60,6 +60,25 @@ void my_write_core(int sig);
 #endif
 
 
+/* print stack traces for all threads */
+void my_pstack();
+
+/* print stack traces for all threads before assert failures */
+#define assert_with_stack_traces(expr) \
+  do { if(!(expr)) { my_pstack(); assert(0); } } while(0)
+
+/* print stack traces for all threads before abort. Please note
+   that when using this with ; in the end, a null statement will
+   be generated in the end. It is fine as long as compilers don't
+   complain.
+ */
+
+#define abort_with_stack_traces() \
+  { my_pstack(); abort(); }
+
+#define assert_0_with_stack_traces() \
+  { my_pstack(); assert(0); }
+
 
 /**
   Async-signal-safe utility functions used by signal handler routines.

mysys/stacktrace.c

@@ -428,6 +428,64 @@ void my_write_core(int sig)
 #endif
 }
 
+
+/* print stack traces for all threads */
+void my_pstack()
+{
+#ifndef __WIN__
+  /* Print out the current stack trace */
+
+  static const int max_num_of_frames = 100;
+  void *stack_trace[max_num_of_frames];
+  int num_of_frames = backtrace(stack_trace, max_num_of_frames);
+  char **frames = backtrace_symbols(stack_trace, num_of_frames);
+  if (!frames)
+  {
+    my_safe_printf_stderr("backtrace_symbols didn't return valid data.");
+    return;
+  }
+
+  my_safe_printf_stderr("\nThe current stack traces:\n");
+  for (int i = 0; i < num_of_frames; ++i)
+  {
+    my_safe_printf_stderr("%s\n", frames[i]);
+  }
+  free(frames);
+
+  /* Print out all the stack traces */
+
+  char cmd[64];
+  pid_t pid = getpid();
+  sprintf(cmd, "pstack %d", pid);
+
+  my_safe_printf_stderr("\nAll stack traces from %s:\n", cmd);
+  FILE *fp = popen(cmd, "r");
+  if (!fp)
+  {
+    my_safe_printf_stderr("Couldn't run \"%s\" by popen.\n", cmd);
+    return;
+  }
+
+  static const size_t max_line_length = 4095;
+  char line_buf[max_line_length + 1];
+  uint output_lines = 0;
+  while (fgets(line_buf, max_line_length, fp))
+  {
+    my_write_stderr(line_buf, strlen(line_buf));
+    ++output_lines;
+  }
+  fclose(fp);
+  if (0 == output_lines)
+  {
+    my_safe_printf_stderr("No stack traces were printed out. "
+                          "This probably was because mysqld didn't have "
+                          "sufficient privileges, and/or some required system "
+                          "settings (suid_dumpable, ulimit, etc...) were not "
+                          "set correctly.\n");
+  }
+#endif  /* not __WIN__ */
+}
+
 #else /* __WIN__*/
 
 #include <dbghelp.h>

sql/sql_select.cc

@@ -44,6 +44,7 @@
 #include "sql_join_buffer.h"     // JOIN_CACHE
 #include "sql_optimizer.h"       // JOIN
 #include "sql_tmp_table.h"       // tmp tables
+#include "my_stacktrace.h"       // abort_with_stack_traces
 
 #ifdef TARGET_OS_LINUX
 #include <sys/syscall.h>
@@ -2954,7 +2955,7 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
       DBUG_PRINT("error",("Table type %d found",tab->type)); /* purecov: deadcode */
       break;					/* purecov: deadcode */
     case JT_UNKNOWN:
-      abort();					/* purecov: deadcode */
+      abort_with_stack_traces();	        /* purecov: deadcode */
     }
     // Materialize derived tables prior to accessing them.
     if (tab->table->pos_in_table_list->uses_materialization())

sql/table_stats.cc

@@ -1,6 +1,7 @@
 #include "sql_base.h"
 #include "sql_show.h"
 #include "my_atomic.h"
+#include "my_stacktrace.h" // abort_with_stack_traces
 
 HASH global_table_stats;
 
@@ -254,7 +255,7 @@ void init_global_table_stats(void)
                 0, 0, (my_hash_get_key)get_key_table_stats,
                 (my_hash_free_key)free_table_stats, 0)) {
     sql_print_error("Initializing global_table_stats failed.");
-    abort();
+    abort_with_stack_traces();
   }
 }
 

storage/innobase/include/ut0dbg.h

@@ -26,10 +26,12 @@ Created 1/30/1994 Heikki Tuuri
 #ifndef ut0dbg_h
 #define ut0dbg_h
 
+#include "my_stacktrace.h" // assert_0_with_stack_traces
+
 #ifdef UNIV_INNOCHECKSUM
 #define ut_a		assert
 #define ut_ad		assert
-#define ut_error	assert(0)
+#define ut_error	assert_0_with_stack_traces()
 #else /* !UNIV_INNOCHECKSUM */
 
 #include "univ.i"
@@ -62,7 +64,7 @@ ut_dbg_assertion_failed(
 	UNIV_COLD MY_ATTRIBUTE((nonnull(2)));
 
 /** Abort the execution. */
-# define UT_DBG_PANIC abort()
+# define UT_DBG_PANIC abort_with_stack_traces()
 
 /** Abort execution if EXPR does not evaluate to nonzero.
 @param EXPR	assertion expression that should hold */

storage/innobase/include/ut0rbt.h

@@ -33,12 +33,13 @@ Created 2007-03-20 Sunny Bains
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include "my_stacktrace.h" //assert_with_stack_traces,assert_0_with_stack_traces
 
 #define	ut_malloc	malloc
 #define	ut_free		free
 #define	ulint		unsigned long
-#define	ut_a(c)		assert(c)
-#define ut_error	assert(0)
+#define	ut_a(c)		assert_with_stack_traces(c)
+#define ut_error	assert_0_with_stack_traces()
 #define	ibool		unsigned int
 #define	TRUE		1
 #define	FALSE		0