add support for always on profiling to zero config (#2538)

* add support for profiler and memory metrics

-Dsplunk.profiler.enabled=true
-Dsplunk.profiler.memory.enabled=true
-Dsplunk.metrics.enabled=true

These are turned off by default, so existing
users will not be affected at this time.

instrumentation/Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get update && \
 WORKDIR /libsplunk
 
 COPY src /libsplunk/src
-COPY testdata/instrumentation.conf /libsplunk/testdata/instrumentation.conf
-COPY testdata/instrumentation-svcname.conf /libsplunk/testdata/instrumentation-svcname.conf
+COPY testdata/instrumentation-default.conf /libsplunk/testdata/instrumentation-default.conf
+COPY testdata/instrumentation-options.conf /libsplunk/testdata/instrumentation-options.conf
 COPY install/instrumentation.conf /libsplunk/install/instrumentation.conf
 COPY Makefile /libsplunk/Makefile

instrumentation/README.md

@@ -38,15 +38,15 @@ The full path to the auto instrumentation JAR (provided by the installer).
 #### `generate_service_name` (optional)
 
 Whether to disable service name generation by the .so. If set to "true" (the default), the .so will attempt to generate
-a service name from the Java command arguments. If set to "false", it will not set the service name, leaving it to be
-generated by the Java auto-instrumentation library.
-
+a service name from the Java command arguments, or provide a hard-coded service name if `service_name` is set. If set to
+"false", it will not set the service name, leaving it to be generated by the Java auto-instrumentation library.
 #### `service_name` (optional)
 
 This is an optional override for the service name that would otherwise be generated by the shared object before Java
 startup. By default, this line is commented out, but can be uncommented to override the generated name. If this
 parameter is set, all instrumented Java applications on this host will have the specified service name (via the
-OTEL_SERVICE_NAME environment variable).
+OTEL_SERVICE_NAME environment variable). If this override is set and `generate_service_name` is also explicitly set to
+`false`, that parameter will win, and the service name will not be set.
 
 ### resource_attributes (optional -- typically set by the installer script)
 
@@ -58,6 +58,35 @@ Java instrumentation jar. Typically, it will be set to something like:
 
 to set the deployment environment for the Splunk backend.
 
+### disable_telemetry (optional)
+
+Set this value to `true` to disable the preloader from sending the `splunk.linux-autoinstr.executions` metric to the
+local collector. Default: `false`.
+
+### generate_service_name (optional)
+
+Set this value to `false` to prevent the preloader from setting the `OTEL_SERVICE_NAME` environment variable.
+Default: `true`. If this value is `false`, the preloader will not set `OTEL_SERVICE_NAME`, and the soon-to-be running
+Java instrumentation library will attempt to set it instead. In the future, this will be the default behavior.
+
+### enable_profiler (optional)
+
+Set this value to `true` to pass `-Dsplunk.profiler.enabled=true` to the starting Java executable, which will enable
+[AlwaysOn CPU Profiling](https://docs.splunk.com/Observability/apm/profiling/get-data-in-profiling.html).
+Default: `false`.
+
+### enable_profiler_memory (optional)
+
+Set this value to `true` to pass `-Dsplunk.profiler.memory.enabled=true` to the starting Java executable, which will
+enable
+[AlwaysOn Memory Profiling](https://docs.splunk.com/Observability/apm/profiling/get-data-in-profiling.html).
+Default: `false`.
+
+### enable_metrics (optional)
+
+Set this value to `true` to pass `-Dsplunk.metrics.enabled=true` to the starting Java executable, which will enable
+[exporting metrics](https://github.com/signalfx/splunk-otel-java/blob/main/docs/metrics.md). Default: `false`.
+
 ### Syntax
 
 To add a comment or comment out a line, start it with a `#`.
@@ -79,7 +108,8 @@ This environment variable contains a `-javaagent` flag set to the full path of t
 
 e.g. `JAVA_TOOL_OPTIONS='-javaagent:/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar'`
 
-This variable is populated by the .so by concatenating the `java_agent_jar` attribute the config to a `-javaagent:` prefix.
+This variable is populated by the .so by concatenating the `java_agent_jar` attribute in the config to a `-javaagent:`
+prefix, and then appending any additional system properties specified in the configuration file.
 
 #### OTEL_SERVICE_NAME
 
@@ -93,7 +123,7 @@ _Meta: link to docs about how service name is used and why it's required._
 #### OTEL_RESOURCE_ATTRIBUTES
 
 This environment variable contains a list of name-value pairs (separated by `=`s) passed on to the Java instrumentation
-jar. 
+jar.
 
 This variable is set directly from the optional `resource_attributes` attribute in the config.
 

instrumentation/install/instrumentation.conf

@@ -1,2 +1,10 @@
-# service_name=default.service
 java_agent_jar=/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar
+#resource_attributes=deployment.environment=my.environment
+#service_name=hardcoded.service
+
+# note: any of the the following lines may be uncommented to override defaults
+#disable_telemetry=true
+#generate_service_name=false
+#enable_profiler=true
+#enable_profiler_memory=true
+#enable_metrics=true

instrumentation/src/args.c

@@ -67,14 +67,14 @@ void add_token(struct tokenset *tks, char *token) {
     }
 }
 
-bool has_token(struct tokenset *tks, char *token) {
+int has_token(struct tokenset *tks, char *token) {
     // not doing a set implementation at this time since size of array is small
     for (int i = 0; i < tks->i; ++i) {
         if (strcmp(tks->tokens[i], token) == 0) {
-            return true;
+            return 1;
         }
     }
-    return false;
+    return 0;
 }
 
 void generate_servicename_from_args(char *dest, char **args, int num_args) {
@@ -154,20 +154,20 @@ void dedupe_hyphenated(char *out, char *str, struct tokenset *pTokenset) {
     }
 }
 
-bool is_unique_path_element(char *path_element) {
+int is_unique_path_element(char *path_element) {
     if (strlen(path_element) == 0) {
-        return false;
+        return 0;
     }
 
     static const char *standard_path_parts[] = {"usr", "local", "bin", "home", "etc", "lib", "opt", ".."};
     static const int num_standard_path_parts = 8;
     for (int i = 0; i < num_standard_path_parts; ++i) {
         const char *part = standard_path_parts[i];
         if (strcmp(part, path_element) == 0) {
-            return false;
+            return 0;
         }
     }
-    return true;
+    return 1;
 }
 
 // removes a .jar suffix/extension from a string if it's long enough
@@ -181,7 +181,7 @@ void truncate_extension(char *str) {
     }
 }
 
-bool is_legal_java_main_class_with_module(const char *str) {
+int is_legal_java_main_class_with_module(const char *str) {
     int num_slashes = 0;
     int num_dots = 0;
     for (int i = 0; str[i] != 0; ++i) {
@@ -193,73 +193,73 @@ bool is_legal_java_main_class_with_module(const char *str) {
         }
     }
     if (num_dots == 0) {
-        return false;
+        return 0;
     }
     if (num_slashes == 0) {
         return is_legal_java_main_class(str);
     } else if (num_slashes > 1) {
-        return false;
+        return 0;
     }
     char *fq_main_package = strdup(str);
     char *module = strsep(&fq_main_package, "/");
 
     if (!is_legal_java_main_class(fq_main_package)) {
-        return false;
+        return 0;
     }
 
     if (!is_legal_module(module)) {
-        return false;
+        return 0;
     }
 
-    return true;
+    return 1;
 }
 
-bool is_legal_java_main_class(const char *str) {
+int is_legal_java_main_class(const char *str) {
     if (strstr(str, ".") == NULL) {
-        return false;
+        return 0;
     }
     char *dup = strdup(str);
     char *prev;
-    while (true) {
+    while (1) {
         char *token = strsep(&dup, ".");
         if (token == NULL) {
             return is_capital_letter(prev[0]);
         }
         if (!is_legal_java_package_element(token)) {
-            return false;
+            return 0;
         }
         prev = token;
     }
 }
 
-bool is_capital_letter(const char ch) {
+int is_capital_letter(const char ch) {
     return ch >= 'A' && ch <= 'Z';
 }
 
-bool is_legal_module(char *module) {
+int is_legal_module(char *module) {
     char *dup = strdup(module);
     char *token;
     while ((token = strsep(&dup, ".")) != NULL) {
         if (!is_legal_java_package_element(token)) {
-            return false;
+            return 0;
         }
     }
-    return true;
+    return 1;
 }
 
 // tests if the parts between the dots in e.g. some.package.MyMain are legal
-bool is_legal_java_package_element(const char *str) {
+int is_legal_java_package_element(const char *str) {
     for (int i = 0;; ++i) {
         char ch = str[i];
         if (ch == 0) {
             break;
         }
         if (i == 0 && ch >= '0' && ch <= '9') {
-            return false;
+            return 0;
         }
         if (ch < '0' || (ch > '9' && ch < 'A') || (ch > 'Z' && ch < '_') || ch == '`' || ch > 'z') {
-            return false;
+            return 0;
         }
     }
-    return true;
+    return 1;
 }

instrumentation/src/args.h

@@ -15,7 +15,7 @@ void init_tokenset(struct tokenset *tks);
 
 void free_tokenset(struct tokenset *tks);
 
-bool has_token(struct tokenset *tks, char *token);
+int has_token(struct tokenset *tks, char *token);
 
 void add_token(struct tokenset *tks, char *token);
 
@@ -25,23 +25,23 @@ void free_cmdline_args(char **args, int num_args);
 
 void generate_servicename_from_args(char *dest, char **args, int num_args);
 
-bool is_legal_java_main_class(const char *str);
+int is_legal_java_main_class(const char *str);
 
-bool is_capital_letter(char ch);
+int is_capital_letter(char ch);
 
-bool is_legal_java_main_class_with_module(const char *str);
+int is_legal_java_main_class_with_module(const char *str);
 
-bool is_legal_module(char *module);
+int is_legal_module(char *module);
 
-bool is_legal_java_package_element(const char *str);
+int is_legal_java_package_element(const char *str);
 
 void transform_multi_jars(char *dest, char *arg, struct tokenset *tks);
 
 void transform_jar_path_elements(char *out, char *path);
 
 void dedupe_hyphenated(char *out, char *str, struct tokenset *pTokenset);
 
-bool is_unique_path_element(char *path_element);
+int is_unique_path_element(char *path_element);
 
 void truncate_extension(char *str);
 

instrumentation/src/cmdline_reader.c

@@ -22,7 +22,7 @@ void cmdline_reader_open(cmdline_reader cr) {
     cr->f = fopen(fname, "r");
 }
 
-bool cmdline_reader_is_eof(cmdline_reader cr) {
+int cmdline_reader_is_eof(cmdline_reader cr) {
     return feof(cr->f) != 0;
 }
 

instrumentation/src/cmdline_reader.h

@@ -1,13 +1,11 @@
 #ifndef INSTRUMENTATION_CMDLINE_READER_H
 #define INSTRUMENTATION_CMDLINE_READER_H
 
-#include <stdbool.h>
-
 typedef struct cmdline_reader_impl *cmdline_reader;
 cmdline_reader new_cmdline_reader();
 cmdline_reader new_test_cmdline_reader(char *cmdline, int len);
 void cmdline_reader_open(cmdline_reader cr);
-bool cmdline_reader_is_eof(cmdline_reader cr);
+int cmdline_reader_is_eof(cmdline_reader cr);
 char cmdline_reader_get_char(cmdline_reader cr);
 void cmdline_reader_close(cmdline_reader cr);
 

instrumentation/src/cmdline_reader_test.c

@@ -31,7 +31,7 @@ cmdline_reader new_test_cmdline_reader(char *cmdline, int size) {
 void cmdline_reader_open(cmdline_reader cr) {
 }
 
-bool cmdline_reader_is_eof(cmdline_reader cr) {
+int cmdline_reader_is_eof(cmdline_reader cr) {
     return cr->i >= cr->size;
 }