add support for profiler and memory metrics

-Dsplunk.profiler.enabled=true -Dsplunk.profiler.memory.enabled=true -Dsplunk.metrics.enabled=true These are turned off by default, so existing users will not be affected at this time.
signalfx · Feb 7, 2023 · de7f531 · de7f531
1 parent f58c454
commit de7f531
Show file tree

Hide file tree

Showing 10 changed files with 224 additions and 86 deletions.
diff --git a/instrumentation/Dockerfile b/instrumentation/Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update && \
 WORKDIR /libsplunk
 
 COPY src /libsplunk/src
-COPY testdata/instrumentation.conf /libsplunk/testdata/instrumentation.conf
-COPY testdata/instrumentation-svcname.conf /libsplunk/testdata/instrumentation-svcname.conf
+COPY testdata/instrumentation-default.conf /libsplunk/testdata/instrumentation-default.conf
+COPY testdata/instrumentation-options.conf /libsplunk/testdata/instrumentation-options.conf
 COPY install/instrumentation.conf /libsplunk/install/instrumentation.conf
 COPY Makefile /libsplunk/Makefile
diff --git a/instrumentation/README.md b/instrumentation/README.md
@@ -58,6 +58,31 @@ Java instrumentation jar. Typically, it will be set to something like:
 
 to set the deployment environment for the Splunk backend.
 
+### disable_telemetry (optional)
+
+Set this value to `true` to disable the preloader from sending the `splunk.linux-autoinstr.executions` metric to the
+local collector. Default: `false`.
+
+### generate_service_name (optional)
+
+Set this value to `false` to prevent the preloader from setting the `OTEL_SERVICE_NAME` environment variable.
+Default: `true`.
+
+### enable_profiler (optional)
+
+Set this value to `true` to pass `-Dsplunk.profiler.enabled=true` to the starting Java executable, which will enable
+[AlwaysOn CPU Profiling](https://docs.splunk.com/Observability/apm/profiling/get-data-in-profiling.html). Default: `false`.
+
+### enable_profiler_memory (optional)
+
+Set this value to `true` to pass `-Dsplunk.profiler.memory.enabled=true` to the starting Java executable, which will enable
+[AlwaysOn Memory Profiling](https://docs.splunk.com/Observability/apm/profiling/get-data-in-profiling.html). Default: `false`.
+
+### enable_metrics (optional)
+
+Set this value to `true` to pass `-Dsplunk.metrics.enabled=true` to the starting Java executable, which will enable
+[exporting metrics](https://github.com/signalfx/splunk-otel-java/blob/main/docs/metrics.md). Default: `false`.
+
 ### Syntax
 
 To add a comment or comment out a line, start it with a `#`.
@@ -79,7 +104,8 @@ This environment variable contains a `-javaagent` flag set to the full path of t
 
 e.g. `JAVA_TOOL_OPTIONS='-javaagent:/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar'`
 
-This variable is populated by the .so by concatenating the `java_agent_jar` attribute the config to a `-javaagent:` prefix.
+This variable is populated by the .so by concatenating the `java_agent_jar` attribute in the config to a `-javaagent:`
+prefix, and then appending any additional system properties specified in the configuration file.
 
 #### OTEL_SERVICE_NAME
 

diff --git a/instrumentation/install/instrumentation.conf b/instrumentation/install/instrumentation.conf
@@ -1,2 +1,10 @@
-# service_name=default.service
 java_agent_jar=/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar
+#resource_attributes=deployment.environment=my.environment
+#service_name=hardcoded.service
+
+# note: any of the the following lines may be uncommented to override defaults
+#disable_telemetry=true
+#generate_service_name=false
+#enable_profiler=true
+#enable_profiler_memory=true
+#enable_metrics=true
diff --git a/instrumentation/src/config.c b/instrumentation/src/config.c
@@ -15,23 +15,28 @@ void read_lines(struct config *cfg, FILE *fp);
 
 void split_on_eq(char *string, struct kv *pair);
 
+void log_config_field(logger log, const char *field, const char *value);
+
 void load_config(logger log, struct config *cfg, char *file_name) {
     read_config_file(log, cfg, file_name);
-    if (cfg->service_name == NULL) {
-        log_debug(log, "service_name not specified in config");
-    }
-    if (cfg->java_agent_jar == NULL) {
-        log_debug(log, "java_agent_jar not specified in config");
-    }
-    if (cfg->resource_attributes == NULL) {
-        log_debug(log, "resource_attributes not specified in config");
-    }
-    if (cfg->disable_telemetry == NULL) {
-        log_debug(log, "disable_telemetry not specified in config");
-    }
-    if (cfg->generate_service_name == NULL) {
-        log_debug(log, "generate_service_name not specified in config");
+    log_config_field(log, "service_name", cfg->service_name);
+    log_config_field(log, "java_agent_jar", cfg->java_agent_jar);
+    log_config_field(log, "resource_attributes", cfg->resource_attributes);
+    log_config_field(log, "disable_telemetry", cfg->disable_telemetry);
+    log_config_field(log, "generate_service_name", cfg->generate_service_name);
+    log_config_field(log, "enable_profiler", cfg->enable_profiler);
+    log_config_field(log, "enable_profiler_memory", cfg->enable_profiler_memory);
+    log_config_field(log, "enable_metrics", cfg->enable_metrics);
+}
+
+void log_config_field(logger log, const char *field, const char *value) {
+    char msg[MAX_LOG_LINE_LEN] = "";
+    if (value == NULL) {
+        snprintf(msg, MAX_LOG_LINE_LEN, "config: %s not specified", field);
+    } else {
+        snprintf(msg, MAX_LOG_LINE_LEN, "config: %s=%s", field, value);
     }
+    log_debug(log, msg);
 }
 
 void read_config_file(logger log, struct config *cfg, char *file_name) {
@@ -68,6 +73,12 @@ void read_lines(struct config *cfg, FILE *fp) {
             cfg->disable_telemetry = strdup(pair.v);
         } else if (streq(pair.k, "generate_service_name")) {
             cfg->generate_service_name = strdup(pair.v);
+        } else if (streq(pair.k, "enable_profiler")) {
+            cfg->enable_profiler = strdup(pair.v);
+        } else if (streq(pair.k, "enable_profiler_memory")) {
+            cfg->enable_profiler_memory = strdup(pair.v);
+        } else if (streq(pair.k, "enable_metrics")) {
+            cfg->enable_metrics = strdup(pair.v);
         }
     }
 }
@@ -77,12 +88,14 @@ void split_on_eq(char *string, struct kv *pair) {
     pair->v = string;
 }
 
-bool str_eq_true(char *v) {
-    return v != NULL && !streq("false", v) && !streq("FALSE", v) && !streq("0", v);
-}
-
-bool str_eq_false(char *v) {
-    return v != NULL && (streq("false", v) || streq("FALSE", v) || streq("0", v));
+bool str_to_bool(char *v, bool defaultVal) {
+    if (v == NULL) {
+        return defaultVal;
+    }
+    if (streq("false", v) || streq("FALSE", v) || streq("0", v)) {
+        return false;
+    }
+    return true;
 }
 
 void free_config(struct config *cfg) {
@@ -101,4 +114,13 @@ void free_config(struct config *cfg) {
     if (cfg->generate_service_name != NULL) {
         free(cfg->generate_service_name);
     }
+    if (cfg->enable_profiler != NULL) {
+        free(cfg->enable_profiler);
+    }
+    if (cfg->enable_profiler_memory != NULL) {
+        free(cfg->enable_profiler_memory);
+    }
+    if (cfg->enable_metrics != NULL) {
+        free(cfg->enable_metrics);
+    }
 }
diff --git a/instrumentation/src/config.h b/instrumentation/src/config.h
@@ -11,13 +11,14 @@ struct config {
     char *resource_attributes;
     char *disable_telemetry;
     char *generate_service_name;
+    char *enable_profiler;
+    char *enable_profiler_memory;
+    char *enable_metrics;
 };
 
 void load_config(logger log, struct config *cfg, char *file_name);
 
-bool str_eq_true(char *v);
-
-bool str_eq_false(char *v);
+bool str_to_bool(char *v, bool);
 
 void free_config(struct config *cfg);
 

diff --git a/instrumentation/src/splunk.c b/instrumentation/src/splunk.c
@@ -8,14 +8,17 @@
 #include <unistd.h>
 #include <stdio.h>
 
-#define ENV_VAR_LEN 512
+#define MAX_ENV_VAR_LEN 512
 #define MAX_CONFIG_ATTR_LEN 256
 #define MAX_CMDLINE_LEN 16000
 #define MAX_ARGS 256
 
 #define JAVA_TOOL_OPTIONS_PREFIX "-javaagent:";
 
 static char *const conf_file = "/usr/lib/splunk-instrumentation/instrumentation.conf";
+static char *const prof_enabled_cmdline_switch = " -Dsplunk.profiler.enabled=true";
+static char *const prof_memory_enabled_cmdline_switch = " -Dsplunk.profiler.memory.enabled=true";
+static char *const metrics_enabled_cmdline_switch = " -Dsplunk.metrics.enabled=true";
 
 extern char *program_invocation_short_name;
 
@@ -73,7 +76,11 @@ void auto_instrument(
             .java_agent_jar = NULL,
             .resource_attributes = NULL,
             .service_name = NULL,
-            .disable_telemetry = NULL
+            .disable_telemetry = NULL,
+            .generate_service_name = NULL,
+            .enable_profiler = NULL,
+            .enable_profiler_memory = NULL,
+            .enable_metrics = NULL
     };
     load_config_func(log, &cfg, conf_file);
     if (cfg.java_agent_jar == NULL) {
@@ -87,22 +94,22 @@ void auto_instrument(
     }
 
     char service_name[MAX_CMDLINE_LEN] = "";
-    if (str_eq_false(cfg.generate_service_name)) {
-        log_debug(log, "service name generation disabled");
-    } else {
+    if (str_to_bool(cfg.generate_service_name, true)) {
         get_service_name(log, cr, &cfg, service_name);
         if (strlen(service_name) == 0) {
             log_info(log, "service name empty, quitting");
             return;
         }
         set_env_var(log, otel_service_name_var, service_name);
+    } else {
+        log_debug(log, "service name generation explicitly disabled");
     }
 
     set_java_tool_options(log, &cfg);
 
     set_env_var_from_attr(log, "resource_attributes", resource_attributes_var, cfg.resource_attributes);
 
-    if (str_eq_true(cfg.disable_telemetry)) {
+    if (str_to_bool(cfg.disable_telemetry, false)) {
         log_info(log, "disabling telemetry as per config");
     } else {
         send_otlp_metric_func(log, service_name);
@@ -148,15 +155,26 @@ void set_env_var(logger log, const char *var_name, const char *value) {
 }
 
 void set_java_tool_options(logger log, struct config *cfg) {
-    char java_tool_options[ENV_VAR_LEN] = JAVA_TOOL_OPTIONS_PREFIX;
+    char java_tool_options[MAX_ENV_VAR_LEN] = JAVA_TOOL_OPTIONS_PREFIX;
     char log_line[MAX_LOG_LINE_LEN] = "";
     size_t jar_path_len = strlen(cfg->java_agent_jar);
     if (jar_path_len > MAX_CONFIG_ATTR_LEN) {
         sprintf(log_line, "jar_path too long: got %zu chars, max %d chars", jar_path_len, MAX_CONFIG_ATTR_LEN);
         log_warning(log, log_line);
         return;
     }
-    strcat(java_tool_options, (*cfg).java_agent_jar);
+    strncat(java_tool_options, cfg->java_agent_jar, MAX_ENV_VAR_LEN - strlen(java_tool_options) - 1);
+
+    if (str_to_bool(cfg->enable_profiler, false)) {
+        strncat(java_tool_options, prof_enabled_cmdline_switch, MAX_ENV_VAR_LEN - strlen(java_tool_options) - 1);
+    }
+    if (str_to_bool(cfg->enable_profiler_memory, false)) {
+        strncat(java_tool_options, prof_memory_enabled_cmdline_switch, MAX_ENV_VAR_LEN - strlen(java_tool_options) - 1);
+    }
+    if (str_to_bool(cfg->enable_metrics, false)) {
+        strncat(java_tool_options, metrics_enabled_cmdline_switch, MAX_ENV_VAR_LEN - strlen(java_tool_options) - 1);
+    }
+
     sprintf(log_line, "setting JAVA_TOOL_OPTIONS='%s'", java_tool_options);
     log_debug(log, log_line);
     setenv(java_tool_options_var, java_tool_options, 0);