Add perf telemetry device

Closes #28

docs/telemetry.rst

@@ -18,6 +18,7 @@ You probably want to gain additional insights from a benchmarking trial. Therefo
     ---------  ---------------------  --------------------------------------------------------------------------------------
     jfr        Flight Recorder        Enables Java Flight Recorder on the benchmark candidate (will only work on Oracle JDK)
     jit        JIT Compiler Profiler  Enables JIT compiler logs.
+    perf       perf stat              Reads CPU PMU counters (beta, only on Linux, requires perf)
 
     Keep in mind that each telemetry device may incur a runtime overhead which can skew results.
 
@@ -40,4 +41,9 @@ The `jit` telemetry device enables JIT compiler logs for the benchmark candidate
 will also contain the disassembled JIT compiler output which can be used for low-level analysis. We recommend to use
 `JITWatch <https://github.com/AdoptOpenJDK/jitwatch>`_ for analysis.
 
-The JITWatch wiki contains `build instructions for hsdis <https://github.com/AdoptOpenJDK/jitwatch/wiki/Building-hsdis>`_.
+The JITWatch wiki contains `build instructions for hsdis <https://github.com/AdoptOpenJDK/jitwatch/wiki/Building-hsdis>`_.
+
+perf
+----
+
+The `perf` telemetry devices runs `perf stat` on each benchmarked node and writes the output to a log file. It can be used to capture low-level CPU statistics. Note that the perf tool, which is only available on Linux, must be installed before using this telemetry device.

esrally/telemetry.py

@@ -236,6 +236,7 @@ def __init__(self, config, metrics_store):
         super().__init__(config, metrics_store)
         self.process = None
         self.node = None
+        self.log = None
 
     @property
     def internal(self):
@@ -254,19 +255,18 @@ def help(self):
         return "Reads CPU PMU counters (beta, only on Linux, requires perf)"
 
     def attach_to_node(self, node):
+        log_root = "%s/%s" % (self._config.opts("system", "track.setup.root.dir"), self._config.opts("benchmarks", "metrics.log.dir"))
+        io.ensure_dir(log_root)
+        log_file = "%s/%s.perf.log" % (log_root, node.node_name)
+
+        logger.info("%s: Writing perf logs to [%s]." % (self.human_name, log_file))
+        print("%s: Writing perf logs to %s" % (self.human_name, log_file))
+
+        self.log = open(log_file, "wb")
+
         self.process = subprocess.Popen(["perf", "stat", "-p %s" % node.process.pid],
-                                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
+                                        stdout=self.log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
         self.node = node
-        t = threading.Thread(target=self.read_data)
-        t.setDaemon(True)
-        t.start()
-
-    def read_data(self):
-        while True:
-            line = self.process.stdout.readline().decode("utf-8")
-            if len(line) == 0:
-                break
-            logger.info("%s: %s" % (self.node.node_name, line.rstrip()))
 
     def detach_from_node(self, node):
         logger.info("Dumping PMU counters for node [%s]" % node.node_name)
@@ -275,6 +275,7 @@ def detach_from_node(self, node):
             self.process.wait(10.0)
         except subprocess.TimeoutExpired:
             logger.warn("perf stat did not terminate")
+        self.log.close()
 
 
 class MergeParts(InternalTelemetryDevice):