Merge PR #87: Add proper support for invocation/iteration

rebench/configurator.py

@@ -25,7 +25,7 @@
 from .model.exp_variables import ExpVariables
 from .model.reporting import Reporting
 from .model.virtual_machine import VirtualMachine
-from .ui import UIError
+from .ui import UIError, escape_braces
 
 
 class _VMFilter(object):
@@ -130,9 +130,10 @@ def load_config(file_name):
             try:
                 validator.validate(raise_exception=True)
             except SchemaError as err:
+                errors = [escape_braces(val_err) for val_err in validator.validation_errors]
                 raise UIError(
                     "Validation of " + file_name + " failed.\n{ind}" +
-                    "{ind}".join(validator.validation_errors) + "\n", err)
+                    "\n{ind}".join(errors) + "\n", err)
             return data
     except IOError as err:
         if err.errno == 2:
@@ -156,8 +157,16 @@ def __init__(self, raw_config, data_store, ui, cli_options=None, cli_reporter=No
         self._data_file = data_file or raw_config.get('standard_data_file', 'rebench.data')
         self._exp_name = exp_name or raw_config.get('standard_experiment', 'all')
 
+        # capture invocation and iteration settings and override when quick is selected
+        invocations = cli_options.invocations if cli_options else None
+        iterations = cli_options.iterations if cli_options else None
+        if cli_options and cli_options.quick:
+            invocations = 1
+            iterations = 1
+
         self._root_run_details = ExpRunDetails.compile(
-            raw_config.get('runs', {}), ExpRunDetails.default())
+            raw_config.get('runs', {}), ExpRunDetails.default(
+                invocations, iterations))
         self._root_reporting = Reporting.compile(
             raw_config.get('reporting', {}), Reporting.empty(cli_reporter), cli_options, ui)
 

rebench/executor.py

@@ -416,21 +416,19 @@ def execute_run(self, run_id):
         if not terminate and self._do_builds:
             self._build_vm_and_suite(run_id)
 
-        stats = StatisticProperties(run_id.get_total_values())
-
         # now start the actual execution
         if not terminate:
             terminate = self._generate_data_point(cmdline, gauge_adapter,
                                                   run_id, termination_check)
-            stats = StatisticProperties(run_id.get_total_values())
 
+        stats = StatisticProperties(run_id.get_total_values())
         if terminate:
             run_id.report_run_completed(stats, cmdline)
             if (not run_id.is_failed() and run_id.min_iteration_time
                     and stats.mean < run_id.min_iteration_time):
                 self._ui.warning(
                     ("{ind}Warning: Low mean run time.\n"
-                     + "{ind}{ind}The mean (%.1f) is lower than min_iteration_time (%d)")
+                     + "{ind}{ind}The mean (%.1f) is lower than min_iteration_time (%d)\n")
                     % (stats.mean, run_id.min_iteration_time), run_id, cmdline)
 
         return terminate
@@ -456,7 +454,6 @@ def _get_gauge_adapter_instance(self, adapter_name):
     def _generate_data_point(self, cmdline, gauge_adapter, run_id,
                              termination_check):
         # execute the external program here
-        run_id.indicate_invocation_start()
 
         try:
             self._ui.debug_output_info("{ind}Starting run\n", run_id, cmdline)
@@ -506,7 +503,7 @@ def _generate_data_point(self, cmdline, gauge_adapter, run_id,
 
     def _eval_output(self, output, run_id, gauge_adapter, cmdline):
         try:
-            data_points = gauge_adapter.parse_data(output, run_id)
+            data_points = gauge_adapter.parse_data(output, run_id, run_id.completed_invocations + 1)
 
             warmup = run_id.warmup_iterations
 
@@ -542,13 +539,17 @@ def _check_termination_condition(run_id, termination_check):
             run_id.get_number_of_data_points())
 
     def execute(self):
-        self._scheduler.execute()
-        successful = True
-        for run in self._runs:
-            run.report_job_completed(self._runs)
-            if run.is_failed():
-                successful = False
-        return successful
+        try:
+            self._scheduler.execute()
+            successful = True
+            for run in self._runs:
+                run.report_job_completed(self._runs)
+                if run.is_failed():
+                    successful = False
+            return successful
+        finally:
+            for run in self._runs:
+                run.close_files()
 
     @property
     def runs(self):

rebench/interop/adapter.py

@@ -38,7 +38,7 @@ def __init__(self, include_faulty):
     def acquire_command(self, command):
         return command
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
         raise NotImplementedError()
 
     def check_for_error(self, line):

rebench/interop/jmh_adapter.py

@@ -33,7 +33,8 @@ class JMHAdapter(GaugeAdapter):
     re_result_line = re.compile(r"^Iteration\s+(\d+):\s+(\d+(?:\.\d+)?)\s+(.+)")
     re_bench = re.compile(r"^# Benchmark: (.+)")
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
 
         for line in data.split("\n"):
@@ -60,8 +61,10 @@ def parse_data(self, data, run_id):
                 criterion = "total"
 
                 point = DataPoint(run_id)
-                point.add_measurement(Measurement(value, unit, run_id, criterion))
+                point.add_measurement(Measurement(invocation, iteration,
+                                                  value, unit, run_id, criterion))
                 data_points.append(point)
+                iteration += 1
 
         if not data_points:
             raise OutputNotParseable(data)

rebench/interop/multivariate_adapter.py

@@ -37,7 +37,8 @@ def __init__(self, include_faulty):
         super(MultivariateAdapter, self).__init__(include_faulty)
         self._other_error_definitions = [re.compile("FAILED")]
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
         current = DataPoint(run_id)
 
@@ -63,14 +64,16 @@ def parse_data(self, data, run_id):
                 else:
                     value = float(value_thing)
 
-                measure = Measurement(value, unit if unit is not None else 'ms',
+                measure = Measurement(invocation, iteration, value,
+                                      unit if unit is not None else 'ms',
                                       run_id, variable)
                 current.add_measurement(measure)
 
                 if cnt is None and measure.is_total():
                     # compatibility for TestVMPerformance
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
 
         if not data_points:
             raise OutputNotParseable(data)

rebench/interop/plain_seconds_log_adapter.py

@@ -41,7 +41,8 @@ def __init__(self, include_faulty):
                                          self.re_NPB_invalid, self.re_incorrect,
                                          self.re_err]
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
         current = DataPoint(run_id)
 
@@ -52,11 +53,12 @@ def parse_data(self, data, run_id):
 
             try:
                 time = float(line) * 1000
-                measure = Measurement(time, 'ms', run_id, 'total')
+                measure = Measurement(invocation, iteration, time, 'ms', run_id, 'total')
                 current.add_measurement(measure)
 
                 data_points.append(current)
                 current = DataPoint(run_id)
+                iteration += 1
             except ValueError:
                 pass  # ignore that line
 

rebench/interop/rebench_log_adapter.py

@@ -45,7 +45,8 @@ def __init__(self, include_faulty):
         self._other_error_definitions = [self.re_NPB_partial_invalid,
                                          self.re_NPB_invalid, self.re_incorrect]
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
         current = DataPoint(run_id)
 
@@ -62,7 +63,7 @@ def parse_data(self, data, run_id):
                     time /= 1000
                 criterion = (match.group(2) or 'total').strip()
 
-                measure = Measurement(time, 'ms', run_id, criterion)
+                measure = Measurement(invocation, iteration, time, 'ms', run_id, criterion)
 
             else:
                 match = self.re_extra_criterion_log_line.match(line)
@@ -71,14 +72,15 @@ def parse_data(self, data, run_id):
                     criterion = match.group(2)
                     unit = match.group(4)
 
-                    measure = Measurement(value, unit, run_id, criterion)
+                    measure = Measurement(invocation, iteration, value, unit, run_id, criterion)
 
             if measure:
                 current.add_measurement(measure)
 
                 if measure.is_total():
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
 
         if not data_points:
             raise OutputNotParseable(data)

rebench/interop/savina_log_adapter.py

@@ -30,17 +30,19 @@ class SavinaLogAdapter(GaugeAdapter):
     re_log_line = re.compile(
         r"^([\w\.]+)\s+Iteration-(?:\d+):\s+([0-9]+\.[0-9]+) ms")
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
 
         for line in data.split("\n"):
             match = self.re_log_line.match(line)
             if match:
                 time = float(match.group(2))
-                measure = Measurement(time, 'ms', run_id, 'total')
+                measure = Measurement(invocation, iteration, time, 'ms', run_id, 'total')
                 current = DataPoint(run_id)
                 current.add_measurement(measure)
                 data_points.append(current)
+                iteration += 1
 
         if not data_points:
             raise OutputNotParseable(data)

rebench/interop/test_adapter.py

@@ -32,9 +32,9 @@ class TestAdapter(GaugeAdapter):
                  45875, 45871, 45869, 45870, 45874]
     index = 0
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, _invocation):
         point = DataPoint(run_id)
-        point.add_measurement(Measurement(self.test_data[self.index], "ms",
+        point.add_measurement(Measurement(1, 1, self.test_data[self.index], "ms",
                                           run_id))
         self.index = (self.index + 1) % len(self.test_data)
         return [point]

rebench/interop/test_vm_adapter.py

@@ -35,7 +35,8 @@ def __init__(self, include_faulty):
         super(TestVMAdapter, self).__init__(include_faulty)
         self._other_error_definitions = [re.compile("FAILED")]
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
         current = DataPoint(run_id)
 
@@ -46,13 +47,15 @@ def parse_data(self, data, run_id):
 
             match = TestVMAdapter.re_time.match(line)
             if match:
-                measure = Measurement(float(match.group(2)), 'ms', run_id,
+                measure = Measurement(invocation, iteration,
+                                      float(match.group(2)), 'ms', run_id,
                                       match.group(1))
                 current.add_measurement(measure)
 
                 if measure.is_total():
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
 
         if not data_points:
             raise OutputNotParseable(data)

rebench/interop/time_adapter.py

@@ -57,9 +57,11 @@ def acquire_command(self, command):
             # TODO: add support for reading out memory info on OS X
             return "/usr/bin/time -p %s" % command
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
         current = DataPoint(run_id)
+        total_measure = None
 
         for line in data.split("\n"):
             if self.check_for_error(line):
@@ -70,14 +72,15 @@ def parse_data(self, data, run_id):
                 match2 = self.re_formatted_time.match(line)
                 if match1:
                     mem_kb = float(match1.group(1))
-                    measure = Measurement(mem_kb, 'kb', run_id, 'MaxRSS')
+                    measure = Measurement(invocation, iteration, mem_kb, 'kb', run_id, 'MaxRSS')
                     current.add_measurement(measure)
                 elif match2:
                     time = float(match2.group(1)) * 1000
-                    measure = Measurement(time, 'ms', run_id, 'total')
+                    measure = Measurement(invocation, iteration, time, 'ms', run_id, 'total')
                     current.add_measurement(measure)
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
             else:
                 match1 = self.re_time.match(line)
                 match2 = self.re_time2.match(line)
@@ -86,15 +89,23 @@ def parse_data(self, data, run_id):
                     criterion = 'total' if match.group(1) == 'real' else match.group(1)
                     time = (float(match.group(2).strip() or 0) * 60 +
                             float(match.group(3))) * 1000
-                    measure = Measurement(time, 'ms', run_id, criterion)
-                    current.add_measurement(measure)
+                    measure = Measurement(invocation, iteration, time, 'ms', run_id, criterion)
+                    if measure.is_total():
+                        total_measure = measure
+                    else:
+                        current.add_measurement(measure)
                 else:
                     measure = None
 
                 if current.number_of_measurements() == 3 and \
                         current.get_total_value() is not None:
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
+
+        if total_measure:
+            current.add_measurement(total_measure)
+            data_points.append(current)
 
         if not data_points:
             raise OutputNotParseable(data)

rebench/interop/validation_log_adapter.py

@@ -47,7 +47,8 @@ def __init__(self, include_faulty):
         self._other_error_definitions = [self.re_NPB_partial_invalid,
                                          self.re_NPB_invalid, self.re_incorrect]
 
-    def parse_data(self, data, run_id):
+    def parse_data(self, data, run_id, invocation):
+        iteration = 1
         data_points = []
         current = DataPoint(run_id)
 
@@ -62,27 +63,34 @@ def parse_data(self, data, run_id):
                 if match.group(5) == "u":
                     time /= 1000
                 criterion = (match.group(2) or 'total').strip()
-                success_measure = Measurement(match.group(6) == "true", 'bool', run_id, 'Success')
-                measure = Measurement(time, 'ms', run_id, criterion)
+                success_measure = Measurement(invocation, iteration,
+                                              match.group(6) == "true", 'bool', run_id, 'Success')
+                measure = Measurement(invocation, iteration, time, 'ms', run_id, criterion)
                 current.add_measurement(success_measure)
                 current.add_measurement(measure)
 
                 if measure.is_total():
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
             else:
                 match = self.re_actors.match(line)
                 if match:
-                    measure1 = Measurement(int(match.group(1)), 'count', run_id, 'Actors')
-                    measure2 = Measurement(int(match.group(2)), 'count', run_id, 'Messages')
-                    measure3 = Measurement(int(match.group(3)), 'count', run_id, 'Promises')
-                    measure4 = Measurement(0, 'ms', run_id, 'total')
+                    measure1 = Measurement(invocation, iteration,
+                                           int(match.group(1)), 'count', run_id, 'Actors')
+                    measure2 = Measurement(invocation, iteration,
+                                           int(match.group(2)), 'count', run_id, 'Messages')
+                    measure3 = Measurement(invocation, iteration,
+                                           int(match.group(3)), 'count', run_id, 'Promises')
+                    measure4 = Measurement(invocation, iteration,
+                                           0, 'ms', run_id, 'total')
                     current.add_measurement(measure1)
                     current.add_measurement(measure2)
                     current.add_measurement(measure3)
                     current.add_measurement(measure4)
                     data_points.append(current)
                     current = DataPoint(run_id)
+                    iteration += 1
 
         if not data_points:
             raise OutputNotParseable(data)