Ensure UTF-8 output in build commands does not break ReBench, Issue #81

rebench/executor.py

@@ -19,19 +19,21 @@
 # THE SOFTWARE.
 from __future__ import with_statement
 
+from codecs import open as open_with_enc
 from collections import deque
 from math import floor
 from multiprocessing import cpu_count
 import os
 import pkgutil
 import random
-from select import select
 import subprocess
 import sys
+from select import select
 from threading import Thread, RLock
 from time import time
 
 from humanfriendly import Spinner
+from humanfriendly.compat import coerce_string
 
 from . import subprocess_with_timeout as subprocess_timeout
 from .interop.adapter import ExecutionDeliveredNoResults
@@ -304,7 +306,8 @@ def _construct_cmdline(self, run_id, gauge_adapter):
     @staticmethod
     def _read(stream):
         data = stream.readline()
-        return data.decode('utf-8')
+        decoded = data.decode('utf-8')
+        return coerce_string(decoded)
 
     def _build_vm_and_suite(self, run_id):
         name = "VM:" + run_id.benchmark.suite.vm.name
@@ -341,7 +344,7 @@ def _execute_build_cmd(self, build_command, name, run_id):
         proc.stdin.close()
 
         if self._build_log:
-            with open(self._build_log, 'a') as log_file:
+            with open_with_enc(self._build_log, 'a', encoding='utf8') as log_file:
                 while True:
                     reads = [proc.stdout.fileno(), proc.stderr.fileno()]
                     ret = select(reads, [], [])

rebench/model/exp_run_details.py

@@ -30,7 +30,7 @@ def compile(cls, config, defaults):
 
         min_iteration_time = none_or_int(config.get('min_iteration_time',
                                                     defaults.min_iteration_time))
-        max_invocation_time = none_or_int(config.get('min_iteration_time',
+        max_invocation_time = none_or_int(config.get('max_invocation_time',
                                                      defaults.max_invocation_time))
 
         parallel_interference_factor = none_or_float(config.get(

rebench/subprocess_with_timeout.py

@@ -9,6 +9,14 @@
 
 import sys
 
+IS_PY3 = None
+
+try:
+    _ = ProcessLookupError
+    IS_PY3 = True
+except NameError:
+    IS_PY3 = False
+
 
 class SubprocessThread(Thread):
 
@@ -127,17 +135,32 @@ def run(args, cwd=None, shell=False, kill_tree=True, timeout=-1,
     return thread.returncode, thread.stdout_result, thread.stderr_result
 
 
+def kill_py2(proc_id):
+    try:
+        kill(proc_id, SIGKILL)
+    except IOError:
+        # it's a race condition, so let's simply ignore it
+        pass
+
+
+def kill_py3(proc_id):
+    try:
+        kill(proc_id, SIGKILL)
+    except ProcessLookupError:  # pylint: disable=undefined-variable
+        # it's a race condition, so let's simply ignore it
+        pass
+
+
 def kill_process(pid, recursively, thread):
     pids = [pid]
     if recursively:
         pids.extend(get_process_children(pid))
 
     for proc_id in pids:
-        try:
-            kill(proc_id, SIGKILL)
-        except ProcessLookupError:  # pylint: disable=undefined-variable
-            # it's a race condition, so let's simply ignore it
-            pass
+        if IS_PY3:
+            kill_py3(proc_id)
+        else:
+            kill_py2(proc_id)
 
     thread.join()
 

rebench/tests/features/issue_81.conf

@@ -0,0 +1,29 @@
+standard_experiment: Test
+standard_data_file: test.data
+
+build_log: build.log
+
+runs:
+    invocations: 2
+    min_iteration_time: 0
+
+benchmark_suites:
+    Suite1:
+        build: ./issue_81_build_suite.sh
+        location: .
+        gauge_adapter: Time
+        command: " "
+        benchmarks:
+          - Bench1
+
+virtual_machines:
+    VM1:
+        build: ./issue_81_build_vm.sh
+        binary: echo foo
+
+experiments:
+    Test:
+        suites:
+          - Suite1
+        executions:
+          - VM1

rebench/tests/features/issue_81_build_suite.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+echo 'A'
+echo '❤'
+echo '囚'
+echo '𐄹'
+echo '💩'
+echo "🧑"
+
+echo 'A'  > /dev/stderr
+echo '❤'  > /dev/stderr
+echo '囚' > /dev/stderr
+echo '𐄹'  > /dev/stderr
+echo '💩'  > /dev/stderr
+echo "🧑"  > /dev/stderr

rebench/tests/features/issue_81_build_vm.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+echo 'A'
+echo '❤'
+echo '囚'
+echo '𐄹'
+echo '💩'
+echo "🧑"
+
+echo 'A'  > /dev/stderr
+echo '❤'  > /dev/stderr
+echo '囚' > /dev/stderr
+echo '𐄹'  > /dev/stderr
+echo '💩'  > /dev/stderr
+echo "🧑"  > /dev/stderr

rebench/tests/features/issue_81_unicode_test.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2018 Stefan Marr <http://www.stefan-marr.de/>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+from __future__ import print_function
+import os
+
+from codecs import open as open_with_enc
+
+from ...configurator     import Configurator, load_config
+from ...executor         import Executor
+from ...persistence      import DataStore
+from ..rebench_test_case import ReBenchTestCase
+
+
+class Issue81UnicodeSuite(ReBenchTestCase):
+
+    def setUp(self):
+        super(Issue81UnicodeSuite, self).setUp()
+        self._set_path(__file__)
+        if os.path.exists(self._path + '/build.log'):
+            os.remove(self._path + '/build.log')
+
+    def test_building(self):
+        cnf = Configurator(load_config(self._path + '/issue_81.conf'), DataStore(),
+                           data_file=self._tmp_file, exp_name='Test')
+        runs = list(cnf.get_runs())
+        runs = sorted(runs, key=lambda e: e.benchmark.name)
+
+        ex = Executor(runs, False, True, build_log=cnf.build_log)
+        ex.execute()
+
+        self.assertEqual("Bench1", runs[0].benchmark.name)
+        self.assertEqual(2, runs[0].get_number_of_data_points())
+
+        self.assertTrue(os.path.exists(self._path + '/build.log'))
+
+        with open_with_enc(self._path + '/build.log', 'r', encoding='utf8') as build_file:
+            log = build_file.read()
+
+        try:
+            unicode_char = unichr(22234)
+        except NameError:
+            unicode_char = chr(22234)
+
+        self.assertGreater(log.find("VM:VM1|ERR:" + unicode_char), -1)
+        self.assertGreater(log.find("VM:VM1|STD:" + unicode_char), -1)
+
+        self.assertGreater(log.find("S:Suite1|ERR:" + unicode_char), -1)
+        self.assertGreater(log.find("S:Suite1|STD:" + unicode_char), -1)
+
+        if os.path.exists(self._path + '/build.log'):
+            os.remove(self._path + '/build.log')