Track executable missing per runId based on fully expanded executable…

… path

We support interpolation of variables in executer path and excutable name, this, need to use the rendered version.

Executor objects may also be instantiated multiple times, which means we can’t really use identity.

Change used vocabulary to remove binary, since we use executable in the config file.

rebench/executor.py

@@ -114,10 +114,10 @@ def _process_remaining_runs(self, runs):
                 completed = False
                 while not completed:
                     completed = self._executor.execute_run(run_id)
-                    self._indicate_progress(completed, run_id)
-                    if run_id.is_binary_missing():
+                    if run_id.executable_missing:
                         remaining_runs = self._executor.without_missing_binaries(
-                            run_id.get_executor(), remaining_runs)
+                            run_id, remaining_runs)
+                    self._indicate_progress(completed, run_id)
             except FailedBuilding:
                 pass
 
@@ -130,13 +130,12 @@ def _process_remaining_runs(self, runs):
             try:
                 run = task_list.popleft()
                 completed = self._executor.execute_run(run)
-                self._indicate_progress(completed, run)
                 if not completed:
                     task_list.append(run)
-                else:
-                    if run.is_binary_missing():
-                        task_list = deque(self._executor.without_missing_binaries(
-                            run.get_executor(), task_list))
+                elif run.executable_missing:
+                    task_list = deque(self._executor.without_missing_binaries(
+                        run, task_list))
+                self._indicate_progress(completed, run)
 
             except FailedBuilding:
                 pass
@@ -150,12 +149,12 @@ def _process_remaining_runs(self, runs):
             run = random.choice(task_list)
             try:
                 completed = self._executor.execute_run(run)
-                self._indicate_progress(completed, run)
                 if completed:
                     task_list.remove(run)
-                    if run.is_binary_missing():
+                    if run.executable_missing:
                         task_list = self._executor.without_missing_binaries(
-                            run.get_executor(), task_list)
+                            run, task_list)
+                self._indicate_progress(completed, run)
 
             except FailedBuilding:
                 task_list.remove(run)
@@ -412,16 +411,16 @@ def process_output(self, name, stdout_result, stderr_result):
                 log_file.write(name + '|ERR:')
                 log_file.write(stderr_result)
 
-    def without_missing_binaries(self, executor, runs):
+    def without_missing_binaries(self, run_exe_missing, runs):
         is_first = True
         remaining_runs = []
         for run in runs:
-            if run.get_executor() is executor:
+            if run.has_same_executable(run_exe_missing):
                 run.fail_immediately()
                 run.report_run_failed(None, None, None)
                 run.report_run_completed(None)
                 if is_first:
-                    self.ui.warning("{ind}Aborting remaining benchmarks using %s." % executor.name)
+                    self.ui.warning("{ind}Aborting remaining benchmarks using %s." % run.executable)
                     is_first = False
             else:
                 remaining_runs.append(run)
@@ -519,7 +518,7 @@ def _keep_alive(seconds):
                        run_id.benchmark.suite.executor.name, return_code, output.strip())
             self.ui.error(msg, run_id, cmdline)
             run_id.report_run_failed(cmdline, return_code, output)
-            run_id.mark_binary_as_missing()
+            run_id.executable_missing = True
             return True
         elif return_code != 0 and not self._include_faulty and not (
                 return_code == subprocess_timeout.E_TIMEOUT and run_id.ignore_timeouts):

rebench/model/executor.py

@@ -60,7 +60,6 @@ def __init__(self, name, path, executable, args, build, description,
            the executor definitions
         """
         self.name = name
-        self.binary_missing = False
         self.path = path
         self.executable = executable
         self.args = args

rebench/model/run_id.py

@@ -44,10 +44,15 @@ def __init__(self, benchmark, cores, input_size, var_value, machine):
 
         self._termination_check = None
         self._cmdline = None
+        self.executable = None
+        self.executable_missing = False
         self.is_failed = True
 
         self._max_invocation = 0
 
+    def has_same_executable(self, other):
+        return self.executable == other.executable
+
     @property
     def warmup_iterations(self):
         return self.benchmark.run_details.warmup
@@ -152,11 +157,6 @@ def indicate_successful_execution(self):
         self.is_failed = False
         self._termination_check.indicate_successful_execution()
 
-    def mark_binary_as_missing(self):
-        self.benchmark.suite.executor.binary_missing = True
-
-    def is_binary_missing(self):
-        return self.benchmark.suite.executor.binary_missing
 
     def add_reporter(self, reporter):
         self._reporters.add(reporter)
@@ -214,9 +214,6 @@ def add_data_point(self, data_point, warmup):
         for persistence in self._persistence:
             persistence.persist_data_point(data_point)
 
-    def get_executor(self):
-        return self.benchmark.suite.executor
-
     def get_number_of_data_points(self):
         return self.statistics.num_samples
 
@@ -294,6 +291,7 @@ def _construct_cmdline(self):
         cmdline = self._expand_vars(cmdline)
 
         self._cmdline = cmdline.strip()
+        self.executable = cmdline.split(' ')[0]
         return self._cmdline
 
     def __eq__(self, other):

rebench/tests/executor_test.py

@@ -59,7 +59,7 @@ def test_broken_command_format_with_ValueError(self):
             ex.execute()
         self.assertIsInstance(err.exception.source_exception, ValueError)
 
-    def _remove_executors_with_missing_binary(self, scheduler):
+    def _remove_executors_with_missing_exe(self, scheduler):
         yaml = load_config(self._path + '/test.conf')
 
         # change config to use executable that doesn't exist
@@ -83,17 +83,17 @@ def _remove_executors_with_missing_binary(self, scheduler):
         ex = Executor(initial_runs, False, self.ui, False, False, scheduler)
         ex.execute()
         self.assertEqual(len(reporter.runs_completed), 28)
-        self.assertEqual(len(reporter.runs_failed), 11)
-        self.assertEqual(len(reporter.runs_failed_without_return_code), 3)
+        self.assertEqual(len(reporter.runs_failed), 1)
+        self.assertEqual(len(reporter.runs_failed_without_return_code), 13)
 
-    def test_remove_executors_with_missing_binary_batch(self):
-        self._remove_executors_with_missing_binary(BatchScheduler)
+    def test_remove_executors_with_missing_exe_batch(self):
+        self._remove_executors_with_missing_exe(BatchScheduler)
 
-    def test_remove_executors_with_missing_binary_round_robin(self):
-        self._remove_executors_with_missing_binary(RoundRobinScheduler)
+    def test_remove_executors_with_missing_exe_round_robin(self):
+        self._remove_executors_with_missing_exe(RoundRobinScheduler)
 
-    def test_remove_executors_with_missing_binary_random(self):
-        self._remove_executors_with_missing_binary(RandomScheduler)
+    def test_remove_executors_with_missing_exe_random(self):
+        self._remove_executors_with_missing_exe(RandomScheduler)
 
     def test_broken_command_format_with_TypeError(self):
         with self.assertRaises(UIError) as err: