From b35123d581cc982d8d168c578e01fd741905ec7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Thu, 14 Apr 2022 20:08:53 +0200 Subject: [PATCH] pmreorder: pass markers to checker Ref: https://github.com/pmem/pmdk/issues/5419 --- src/tools/pmreorder/binaryoutputhandler.py | 24 +++----- src/tools/pmreorder/consistencycheckwrap.py | 4 +- src/tools/pmreorder/markerparser.py | 7 +-- src/tools/pmreorder/memoryoperations.py | 28 ++++----- src/tools/pmreorder/opscontext.py | 19 +++++-- src/tools/pmreorder/pmreorder.py | 3 +- src/tools/pmreorder/reorderengines.py | 6 +- src/tools/pmreorder/statemachine.py | 63 +++++++-------------- 8 files changed, 60 insertions(+), 94 deletions(-) diff --git a/src/tools/pmreorder/binaryoutputhandler.py b/src/tools/pmreorder/binaryoutputhandler.py index d3187893e50..d43f65b2682 100644 --- a/src/tools/pmreorder/binaryoutputhandler.py +++ b/src/tools/pmreorder/binaryoutputhandler.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation from loggingfacility import LoggingBase from reorderexceptions import InconsistentFileException @@ -73,15 +73,11 @@ def do_store(self, store_op): store_ok = False for i, bf in enumerate(self._files): if utils.range_cmp(store_op, bf) == 0: - self._logger.debug( - "Doing store in file no. {0}: {1}".format(i, bf) - ) + self._logger.debug("Doing store in file no. {0}: {1}".format(i, bf)) bf.do_store(store_op) store_ok = True if not store_ok: - raise OSError( - "No suitable file found for store {}".format(store_op) - ) + raise OSError("No suitable file found for store {}".format(store_op)) def do_revert(self, store_op): """ @@ -101,9 +97,7 @@ def do_revert(self, store_op): bf.do_revert(store_op) revert_ok = True if not revert_ok: - raise OSError( - "No suitable file found for store {}".format(store_op) - ) + raise OSError("No suitable file found for store {}".format(store_op)) def check_consistency(self): """ @@ -114,9 +108,7 @@ def check_consistency(self): """ for bf in self._files: if not bf.check_consistency(): - raise InconsistentFileException( - "File {} inconsistent".format(bf) - ) + raise InconsistentFileException("File {} inconsistent".format(bf)) class BinaryFile(utils.Rangeable): @@ -163,9 +155,7 @@ def __init__(self, file_name, map_base, size, checker, logger=None): def __str__(self): return "{0} (base: {1}, size: {2})".format( - self._file_name, - hex(self._map_base), - hex(self._map_max - self._map_base) + self._file_name, hex(self._map_base), hex(self._map_max - self._map_base) ) def do_store(self, store_op): @@ -185,7 +175,7 @@ def do_store(self, store_op): self._logger.debug( "do_store: old_value: {0}, new_value: {1}".format( hex(int.from_bytes(store_op.old_value, byteorder=byteorder)), - hex(int.from_bytes(store_op.new_value, byteorder=byteorder)) + hex(int.from_bytes(store_op.new_value, byteorder=byteorder)), ) ) diff --git a/src/tools/pmreorder/consistencycheckwrap.py b/src/tools/pmreorder/consistencycheckwrap.py index 3ef3314f78f..fe48f11f1e5 100644 --- a/src/tools/pmreorder/consistencycheckwrap.py +++ b/src/tools/pmreorder/consistencycheckwrap.py @@ -66,9 +66,7 @@ def check_consistency(self, filename): """ if self._lib_func is None: raise RuntimeError( - "Consistency check function {} not loaded".format( - self._lib_func_name - ) + "Consistency check function {} not loaded".format(self._lib_func_name) ) self._logger.debug( diff --git a/src/tools/pmreorder/markerparser.py b/src/tools/pmreorder/markerparser.py index 5e558309bc9..3c804eb0b9d 100644 --- a/src/tools/pmreorder/markerparser.py +++ b/src/tools/pmreorder/markerparser.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation import os @@ -26,8 +26,7 @@ def marker_file_parser(self, macros): print( "Invalid config macros file format: ", macros, - 'Use: {"MARKER_NAME1"="ENGINE_TYPE1",' - '"MARKER_NAME2"="ENGINE_TYPE2"}' + 'Use: {"MARKER_NAME1"="ENGINE_TYPE1",' '"MARKER_NAME2"="ENGINE_TYPE2"}', ) return markers @@ -45,7 +44,7 @@ def marker_cli_parser(self, macros): print( "Invalid extended macros format: ", macros, - "Use: MARKER_NAME1=ENGINE_TYPE1,MARKER_NAME2=ENGINE_TYPE2" + "Use: MARKER_NAME1=ENGINE_TYPE1,MARKER_NAME2=ENGINE_TYPE2", ) def get_markers(self, markerset): diff --git a/src/tools/pmreorder/memoryoperations.py b/src/tools/pmreorder/memoryoperations.py index 4b25b2dc99d..2d3dddfab8b 100644 --- a/src/tools/pmreorder/memoryoperations.py +++ b/src/tools/pmreorder/memoryoperations.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation from utils import Rangeable from utils import range_cmp @@ -71,9 +71,7 @@ def __init__(self, values): # calculate the offset given the registered file mapping self.address = int(params[1], 16) self.size = int(params[3], 16) - self.new_value = int(params[2], 16).to_bytes( - self.size, byteorder=byteorder - ) + self.new_value = int(params[2], 16).to_bytes(self.size, byteorder=byteorder) if len(params) > 4: self.trace = StackTrace(params[4:]) else: @@ -86,13 +84,11 @@ def __init__(self, values): self.flushed = False def __str__(self): - return ( - "Store: addr: {0}, size: {1}, val: {2}, stack trace: {3}".format( - hex(self.address), - hex(self.size), - hex(int.from_bytes(self.new_value, byteorder=byteorder)), - self.trace, - ) + return "Store: addr: {0}, size: {1}, val: {2}, stack trace: {3}".format( + hex(self.address), + hex(self.size), + hex(int.from_bytes(self.new_value, byteorder=byteorder)), + self.trace, ) def get_base_address(self): @@ -171,9 +167,7 @@ def __init__(self, values): self._size = int(params[2], 16) def __str__(self): - return "Flush: addr: {0} size: {1}".format( - hex(self._address), hex(self._size) - ) + return "Flush: addr: {0} size: {1}".format(hex(self._address), hex(self._size)) def is_in_flush(self, store_op): """ @@ -463,10 +457,8 @@ def __init__(self, values): self.offset = int(params[4], 16) def __str__(self): - return ( - "Register_file: name: {0} addr: {1} size: {2} offset: {3}".format( - self.name, hex(self.address), hex(self.size), hex(self.offset) - ) + return "Register_file: name: {0} addr: {1} size: {2} offset: {3}".format( + self.name, hex(self.address), hex(self.size), hex(self.offset) ) class Factory: diff --git a/src/tools/pmreorder/opscontext.py b/src/tools/pmreorder/opscontext.py index e360d7dde98..4adc169847c 100644 --- a/src/tools/pmreorder/opscontext.py +++ b/src/tools/pmreorder/opscontext.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation from operationfactory import OperationFactory from binaryoutputhandler import BinaryOutputHandler @@ -56,19 +56,30 @@ def extract_operations(self): :return: list of subclasses of :class:`memoryoperations.BaseOperation` """ + enumerated_ops = list(enumerate(self._operations)) + markers = list( + filter( + lambda e: e[1].endswith(".BEGIN") or e[1].endswith(".END"), + enumerated_ops, + ) + ) + operation_ids = list(enumerated_ops) + stop_index = start_index = 0 - for i, elem in enumerate(self._operations): + for i, elem in enumerated_ops: if "START" in elem: start_index = i elif "STOP" in elem: stop_index = i - return list( + operations = list( map( OperationFactory.create_operation, - self._operations[start_index + 1: stop_index], + self._operations[start_index + 1 : stop_index], repeat(self.markers), repeat(self.stack_engines), ) ) + + return (operations, operation_ids, markers) diff --git a/src/tools/pmreorder/pmreorder.py b/src/tools/pmreorder/pmreorder.py index 2b8777fb3e5..76a71425262 100644 --- a/src/tools/pmreorder/pmreorder.py +++ b/src/tools/pmreorder/pmreorder.py @@ -103,7 +103,8 @@ def main(): # init and run the state machine a = statemachine.StateMachine(statemachine.InitState(context)) - if a.run_all(context.extract_operations()) is False: + operations, operation_ids, markers = context.extract_operations() + if a.run_all(operations, operation_ids, markers) is False: sys.exit(1) diff --git a/src/tools/pmreorder/reorderengines.py b/src/tools/pmreorder/reorderengines.py index 199c9701ada..532433c4bd6 100644 --- a/src/tools/pmreorder/reorderengines.py +++ b/src/tools/pmreorder/reorderengines.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation from itertools import combinations @@ -302,9 +302,7 @@ def generate_sequence(self, store_list): population_size = len(population) for elem in sample( population, - self._max_seq - if self._max_seq <= population_size - else population_size, + self._max_seq if self._max_seq <= population_size else population_size, ): yield elem diff --git a/src/tools/pmreorder/statemachine.py b/src/tools/pmreorder/statemachine.py index fbe7f45fa7f..7b2747e7b60 100644 --- a/src/tools/pmreorder/statemachine.py +++ b/src/tools/pmreorder/statemachine.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation import memoryoperations as memops import reorderengines from reorderexceptions import InconsistentFileException from reorderexceptions import NotSupportedOperationException +import os class State: @@ -165,42 +166,26 @@ def substitute_reorder(self, order_ops): """ if isinstance(order_ops, memops.ReorderFull): self._context.reorder_engine = reorderengines.FullReorderEngine() - self._context.test_on_barrier = ( - self._context.reorder_engine.test_on_barrier - ) + self._context.test_on_barrier = self._context.reorder_engine.test_on_barrier elif isinstance(order_ops, memops.ReorderPartial): # TODO add macro in valgrind or # parameter inside the tool to support parameters? - self._context.reorder_engine = ( - reorderengines.RandomPartialReorderEngine(3) - ) - self._context.test_on_barrier = ( - self._context.reorder_engine.test_on_barrier - ) + self._context.reorder_engine = reorderengines.RandomPartialReorderEngine(3) + self._context.test_on_barrier = self._context.reorder_engine.test_on_barrier elif isinstance(order_ops, memops.ReorderAccumulative): - self._context.reorder_engine = ( - reorderengines.AccumulativeReorderEngine() - ) - self._context.test_on_barrier = ( - self._context.reorder_engine.test_on_barrier - ) + self._context.reorder_engine = reorderengines.AccumulativeReorderEngine() + self._context.test_on_barrier = self._context.reorder_engine.test_on_barrier elif isinstance(order_ops, memops.ReorderReverseAccumulative): self._context.reorder_engine = ( reorderengines.AccumulativeReverseReorderEngine() ) - self._context.test_on_barrier = ( - self._context.reorder_engine.test_on_barrier - ) + self._context.test_on_barrier = self._context.reorder_engine.test_on_barrier elif isinstance(order_ops, memops.NoReorderDoCheck): self._context.reorder_engine = reorderengines.NoReorderEngine() - self._context.test_on_barrier = ( - self._context.reorder_engine.test_on_barrier - ) + self._context.test_on_barrier = self._context.reorder_engine.test_on_barrier elif isinstance(order_ops, memops.NoReorderNoCheck): self._context.reorder_engine = reorderengines.NoCheckerEngine() - self._context.test_on_barrier = ( - self._context.reorder_engine.test_on_barrier - ) + self._context.test_on_barrier = self._context.reorder_engine.test_on_barrier elif isinstance(order_ops, memops.ReorderDefault): self._context.reorder_engine = self._context.default_engine self._context.test_on_barrier = self._context.default_barrier @@ -232,9 +217,7 @@ def reg_file(self, file_op): :type file_op: memoryoperations.Register_file :return: None """ - self._context.file_handler.add_file( - file_op.name, file_op.address, file_op.size - ) + self._context.file_handler.add_file(file_op.name, file_op.address, file_op.size) def move_inner_state(self, in_op): """ @@ -249,10 +232,7 @@ def move_inner_state(self, in_op): """ if isinstance(in_op, memops.Store) and self._inner_state == "init": self._inner_state = "dirty" - elif ( - isinstance(in_op, memops.FlushBase) - and self._inner_state == "dirty" - ): + elif isinstance(in_op, memops.FlushBase) and self._inner_state == "dirty": self._inner_state = "flush" elif isinstance(in_op, memops.Fence) and self._inner_state == "flush": self._inner_state = "fence" @@ -308,23 +288,17 @@ def run(self, in_op): flushed_stores = list(filter(lambda x: x.flushed, self._ops_list)) # not-flushed stores should be passed to next state - State.trans_stores = list( - filter(lambda x: x.flushed is False, self._ops_list) - ) + State.trans_stores = list(filter(lambda x: x.flushed is False, self._ops_list)) if self._context.test_on_barrier: self._context.logger.debug( - "Current reorder engine: {}".format( - self._context.reorder_engine - ) + "Current reorder engine: {}".format(self._context.reorder_engine) ) for i, seq in enumerate( self._context.reorder_engine.generate_sequence(flushed_stores) ): self._context.logger.debug( - "NEXT Sequence (no. {0}) with length: {1}".format( - i, len(seq) - ) + "NEXT Sequence (no. {0}) with length: {1}".format(i, len(seq)) ) for j, op in enumerate(seq): self._context.logger.debug( @@ -371,7 +345,7 @@ def __init__(self, init_state): """ self._curr_state = init_state - def run_all(self, operations): + def run_all(self, operations, operation_ids, markers): """ Starts the state machine. @@ -381,8 +355,11 @@ def run_all(self, operations): :return: None """ all_consistent = True - for ops in operations: + for ops, ops_id in zip(operations, operation_ids): + print(ops_id[0]) self._curr_state._context.logger.info(ops) + markers_to_pass = [m[1] for m in markers if m[0] < ops_id[0]] + os.environ["PMREORDER_MARKERS"] = "|".join(markers_to_pass) self._curr_state = self._curr_state.next(ops) check = self._curr_state.run(ops) if check is False: