pmreorder: pass markers to checker

Ref: pmem#5419

src/tools/pmreorder/binaryoutputhandler.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2018-2021, Intel Corporation
+# Copyright 2018-2022, Intel Corporation
 
 from loggingfacility import LoggingBase
 from reorderexceptions import InconsistentFileException
@@ -74,14 +74,12 @@ def do_store(self, store_op):
         for i, bf in enumerate(self._files):
             if utils.range_cmp(store_op, bf) == 0:
                 self._logger.debug(
-                    "Doing store in file no. {0}: {1}".format(i, bf)
-                )
+                    "Doing store in file no. {0}: {1}".format(i, bf))
                 bf.do_store(store_op)
                 store_ok = True
         if not store_ok:
             raise OSError(
-                "No suitable file found for store {}".format(store_op)
-            )
+                "No suitable file found for store {}".format(store_op))
 
     def do_revert(self, store_op):
         """
@@ -102,8 +100,7 @@ def do_revert(self, store_op):
                 revert_ok = True
         if not revert_ok:
             raise OSError(
-                "No suitable file found for store {}".format(store_op)
-            )
+                "No suitable file found for store {}".format(store_op))
 
     def check_consistency(self):
         """
@@ -115,8 +112,7 @@ def check_consistency(self):
         for bf in self._files:
             if not bf.check_consistency():
                 raise InconsistentFileException(
-                    "File {} inconsistent".format(bf)
-                )
+                    "File {} inconsistent".format(bf))
 
 
 class BinaryFile(utils.Rangeable):
@@ -163,9 +159,8 @@ def __init__(self, file_name, map_base, size, checker, logger=None):
 
     def __str__(self):
         return "{0} (base: {1}, size: {2})".format(
-            self._file_name,
-            hex(self._map_base),
-            hex(self._map_max - self._map_base)
+            self._file_name, hex(self._map_base), hex(
+                self._map_max - self._map_base)
         )
 
     def do_store(self, store_op):
@@ -185,7 +180,7 @@ def do_store(self, store_op):
         self._logger.debug(
             "do_store: old_value: {0}, new_value: {1}".format(
                 hex(int.from_bytes(store_op.old_value, byteorder=byteorder)),
-                hex(int.from_bytes(store_op.new_value, byteorder=byteorder))
+                hex(int.from_bytes(store_op.new_value, byteorder=byteorder)),
             )
         )
 

src/tools/pmreorder/consistencycheckwrap.py

@@ -66,9 +66,8 @@ def check_consistency(self, filename):
         """
         if self._lib_func is None:
             raise RuntimeError(
-                "Consistency check function {} not loaded".format(
-                    self._lib_func_name
-                )
+                "Consistency check function {} not loaded"
+                .format(self._lib_func_name)
             )
 
         self._logger.debug(

src/tools/pmreorder/markerparser.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2018-2021, Intel Corporation
+# Copyright 2018-2022, Intel Corporation
 
 
 import os
@@ -27,7 +27,7 @@ def marker_file_parser(self, macros):
                 "Invalid config macros file format: ",
                 macros,
                 'Use: {"MARKER_NAME1"="ENGINE_TYPE1",'
-                '"MARKER_NAME2"="ENGINE_TYPE2"}'
+                '"MARKER_NAME2"="ENGINE_TYPE2"}',
             )
 
         return markers
@@ -45,7 +45,7 @@ def marker_cli_parser(self, macros):
             print(
                 "Invalid extended macros format: ",
                 macros,
-                "Use: MARKER_NAME1=ENGINE_TYPE1,MARKER_NAME2=ENGINE_TYPE2"
+                "Use: MARKER_NAME1=ENGINE_TYPE1,MARKER_NAME2=ENGINE_TYPE2",
             )
 
     def get_markers(self, markerset):

src/tools/pmreorder/memoryoperations.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2018-2021, Intel Corporation
+# Copyright 2018-2022, Intel Corporation
 
 from utils import Rangeable
 from utils import range_cmp
@@ -72,8 +72,7 @@ def __init__(self, values):
         self.address = int(params[1], 16)
         self.size = int(params[3], 16)
         self.new_value = int(params[2], 16).to_bytes(
-            self.size, byteorder=byteorder
-        )
+            self.size, byteorder=byteorder)
         if len(params) > 4:
             self.trace = StackTrace(params[4:])
         else:
@@ -86,13 +85,12 @@ def __init__(self, values):
         self.flushed = False
 
     def __str__(self):
-        return (
-            "Store: addr: {0}, size: {1}, val: {2}, stack trace: {3}".format(
-                hex(self.address),
-                hex(self.size),
-                hex(int.from_bytes(self.new_value, byteorder=byteorder)),
-                self.trace,
-            )
+        return "Store: addr: {0}, size: {1}, val: {2}, stack trace: \
+            {3}".format(
+            hex(self.address),
+            hex(self.size),
+            hex(int.from_bytes(self.new_value, byteorder=byteorder)),
+            self.trace,
         )
 
     def get_base_address(self):
@@ -171,9 +169,8 @@ def __init__(self, values):
         self._size = int(params[2], 16)
 
     def __str__(self):
-        return "Flush: addr: {0} size: {1}".format(
-            hex(self._address), hex(self._size)
-        )
+        return "Flush: addr: {0} size: {1}".format(hex(self._address),
+                                                   hex(self._size))
 
     def is_in_flush(self, store_op):
         """
@@ -463,10 +460,9 @@ def __init__(self, values):
         self.offset = int(params[4], 16)
 
     def __str__(self):
-        return (
-            "Register_file: name: {0} addr: {1} size: {2} offset: {3}".format(
-                self.name, hex(self.address), hex(self.size), hex(self.offset)
-            )
+        return "Register_file: name: {0} addr: {1} size: {2}\
+        offset: {3}".format(
+            self.name, hex(self.address), hex(self.size), hex(self.offset)
         )
 
     class Factory:

src/tools/pmreorder/opscontext.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2018-2021, Intel Corporation
+# Copyright 2018-2022, Intel Corporation
 
 from operationfactory import OperationFactory
 from binaryoutputhandler import BinaryOutputHandler
@@ -56,19 +56,30 @@ def extract_operations(self):
 
         :return: list of subclasses of :class:`memoryoperations.BaseOperation`
         """
+        enumerated_ops = list(enumerate(self._operations))
+        markers = list(
+            filter(
+                lambda e: e[1].endswith(".BEGIN") or e[1].endswith(".END"),
+                enumerated_ops,
+            )
+        )
+        operation_ids = list(enumerated_ops)
+
         stop_index = start_index = 0
 
-        for i, elem in enumerate(self._operations):
+        for i, elem in enumerated_ops:
             if "START" in elem:
                 start_index = i
             elif "STOP" in elem:
                 stop_index = i
 
-        return list(
+        operations = list(
             map(
                 OperationFactory.create_operation,
                 self._operations[start_index + 1: stop_index],
                 repeat(self.markers),
                 repeat(self.stack_engines),
             )
         )
+
+        return (operations, operation_ids, markers)

src/tools/pmreorder/pmreorder.py

@@ -103,7 +103,8 @@ def main():
 
     # init and run the state machine
     a = statemachine.StateMachine(statemachine.InitState(context))
-    if a.run_all(context.extract_operations()) is False:
+    operations, operation_ids, markers = context.extract_operations()
+    if a.run_all(operations, operation_ids, markers) is False:
         sys.exit(1)
 
 

src/tools/pmreorder/reorderengines.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2018-2021, Intel Corporation
+# Copyright 2018-2022, Intel Corporation
 
 
 from itertools import combinations
@@ -302,9 +302,8 @@ def generate_sequence(self, store_list):
         population_size = len(population)
         for elem in sample(
             population,
-            self._max_seq
-            if self._max_seq <= population_size
-            else population_size,
+            self._max_seq if self._max_seq <= population_size
+                else population_size,
         ):
             yield elem
 

src/tools/pmreorder/statemachine.py

@@ -1,10 +1,11 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2018-2021, Intel Corporation
+# Copyright 2018-2022, Intel Corporation
 
 import memoryoperations as memops
 import reorderengines
 from reorderexceptions import InconsistentFileException
 from reorderexceptions import NotSupportedOperationException
+import os
 
 
 class State:
@@ -158,7 +159,6 @@ def run(self, in_op):
     def substitute_reorder(self, order_ops):
         """
         Changes the reordering engine based on the log marker class.
-
         :param order_ops: The reordering marker class.
         :type order_ops: subclass of :class:`memoryoperations.ReorderBase`
         :return: None
@@ -232,9 +232,8 @@ def reg_file(self, file_op):
         :type file_op: memoryoperations.Register_file
         :return: None
         """
-        self._context.file_handler.add_file(
-            file_op.name, file_op.address, file_op.size
-        )
+        self._context.file_handler.add_file(file_op.name,
+                                            file_op.address, file_op.size)
 
     def move_inner_state(self, in_op):
         """
@@ -249,10 +248,8 @@ def move_inner_state(self, in_op):
         """
         if isinstance(in_op, memops.Store) and self._inner_state == "init":
             self._inner_state = "dirty"
-        elif (
-            isinstance(in_op, memops.FlushBase)
-            and self._inner_state == "dirty"
-        ):
+        elif isinstance(in_op, memops.FlushBase) and \
+                self._inner_state == "dirty":
             self._inner_state = "flush"
         elif isinstance(in_op, memops.Fence) and self._inner_state == "flush":
             self._inner_state = "fence"
@@ -308,23 +305,18 @@ def run(self, in_op):
         flushed_stores = list(filter(lambda x: x.flushed, self._ops_list))
 
         # not-flushed stores should be passed to next state
-        State.trans_stores = list(
-            filter(lambda x: x.flushed is False, self._ops_list)
-        )
+        State.trans_stores = list(filter(lambda x: x.flushed is False,
+                                         self._ops_list))
 
         if self._context.test_on_barrier:
-            self._context.logger.debug(
-                "Current reorder engine: {}".format(
-                    self._context.reorder_engine
-                )
-            )
+            self._context.logger.debug("Current reorder engine: {}"
+                                       .format(self._context.reorder_engine))
             for i, seq in enumerate(
                 self._context.reorder_engine.generate_sequence(flushed_stores)
             ):
                 self._context.logger.debug(
-                    "NEXT Sequence (no. {0}) with length: {1}".format(
-                        i, len(seq)
-                    )
+                    "NEXT Sequence (no. {0}) with length: \
+                        {1}".format(i, len(seq))
                 )
                 for j, op in enumerate(seq):
                     self._context.logger.debug(
@@ -371,7 +363,7 @@ def __init__(self, init_state):
         """
         self._curr_state = init_state
 
-    def run_all(self, operations):
+    def run_all(self, operations, operation_ids, markers):
         """
         Starts the state machine.
 
@@ -381,8 +373,10 @@ def run_all(self, operations):
         :return: None
         """
         all_consistent = True
-        for ops in operations:
+        for ops, ops_id in zip(operations, operation_ids):
             self._curr_state._context.logger.info(ops)
+            markers_to_pass = (m[1] for m in markers if m[0] < ops_id[0])
+            os.environ["PMREORDER_MARKERS"] = "|".join(markers_to_pass)
             self._curr_state = self._curr_state.next(ops)
             check = self._curr_state.run(ops)
             if check is False: