Merge branch 'feature/machine_learning' of github.com:emsec/hal into …

…feature/machine_learning

.clang-format

@@ -32,6 +32,7 @@ BraceWrapping:
   BeforeCatch:     true
   BeforeElse:      true
   IndentBraces:    false
+InsertBraces: true
 BreakBeforeBinaryOperators: NonAssignment
 BreakBeforeBraces: Custom
 # BreakBeforeInheritanceComma: false
@@ -90,7 +91,7 @@ SpacesInContainerLiterals: false
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
-Standard:        Cpp11
+Standard:        c++17
 TabWidth:        4
 UseTab:          Never
 ...

.github/workflows/macOS.yml

@@ -89,12 +89,18 @@ jobs:
         run: |
           mkdir -p build
           cd build
-          export PATH="/usr/local/opt/qt@5/bin:$PATH"
-          ls ..
-          cmake -G Ninja .. -DQt5_DIR=/usr/local/opt/qt@5/lib/cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_ALL_PLUGINS=ON -DBUILD_TESTS=ON -DPL_GUI=ON -DCMAKE_C_COMPILER=/usr/local/opt/llvm@14/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm@14/bin/clang++
+          export PATH="$(brew --prefix qt@5)/bin:$PATH"
+          brew ls llvm@14
+          brew --prefix llvm@14
+          ls -l /usr/local
+          ls -l /opt/homebrew/opt
+          ls -l /opt/homebrew/opt/llvm@14
+          ls -l /opt/homebrew/opt/llvm@14/bin
+          ls -l /opt/homebrew/opt/llvm@14/bin/clang
+          cmake -G Ninja .. -DQt5_DIR="$(brew --prefix qt@5)/lib/cmake" -DCMAKE_BUILD_TYPE=Debug -DBUILD_ALL_PLUGINS=ON -DBUILD_TESTS=ON -DPL_GUI=ON -DCMAKE_C_COMPILER=$(brew --prefix llvm@14)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm@14)/bin/clang++
         env:
-          LDFLAGS: "-L/usr/local/opt/qt@5/lib -L/usr/local/opt/llvm@14/lib -Wl,-rpath,/usr/local/opt/llvm@14/lib"
-          CPPFLAGS: "-I/usr/local/opt/llvm@14/include"
+          LDFLAGS: "-L$(brew --prefix qt@5)/lib -L$(brew --prefix llvm@14)/lib -Wl,-rpath,$(brew --prefix llvm@14)/lib"
+          CPPFLAGS: "-I$(brew --prefix llvm@14)/include"
           HAL_BASE_PATH: ${{runner.workspace}}/hal/build
           CCACHE_DIR: ${{runner.workspace}}/.ccache
           CCACHE_COMPRESS: true
@@ -107,12 +113,12 @@ jobs:
         # Execute the build.  You can specify a specific target with "--target <NAME>"
         run: |
           cd build
-          export PATH="/usr/local/opt/qt/bin:$PATH"
+          export PATH="$(brew --prefix qt@5)/bin:$PATH"
           export BUILD_TYPE=Debug
           cmake --build . --target all --clean-first --config $BUILD_TYPE
         env:
-          LDFLAGS: "-L/usr/local/opt/qt@5/lib -L/usr/local/opt/llvm@14/lib -Wl,-rpath,/usr/local/opt/llvm@14/lib"
-          CPPFLAGS: "-I/usr/local/opt/llvm@14/include"
+          LDFLAGS: "-L$(brew --prefix qt@5)/lib -L$(brew --prefix llvm@14)/lib -Wl,-rpath,$(brew --prefix llvm@14)/lib"
+          CPPFLAGS: "-I$(brew --prefix llvm@14)/include"
           HAL_BASE_PATH: ${{runner.workspace}}/hal/build
           CCACHE_DIR: ${{runner.workspace}}/.ccache
           CCACHE_COMPRESS: true
@@ -133,8 +139,8 @@ jobs:
         #  ninja -v hal_coverage
         # bash <(curl -s https://codecov.io/bash) -f hal_coverage.info.cleaned || echo "Codecov did not collect coverage reports"
         env:
-          LDFLAGS: "-L/usr/local/opt/qt@5/lib -L/usr/local/opt/llvm@14/lib -Wl,-rpath,/usr/local/opt/llvm@14/lib"
-          CPPFLAGS: "-I/usr/local/opt/llvm@14/include"
+          LDFLAGS: "-L$(brew --prefix qt@5)/lib -L$(brew --prefix llvm@14)/lib -Wl,-rpath,$(brew --prefix llvm@14)/lib"
+          CPPFLAGS: "-I$(brew --prefix llvm@14)/include"
           HAL_BASE_PATH: ${{runner.workspace}}/hal/build
           CCACHE_DIR: ${{runner.workspace}}/.ccache
           CCACHE_COMPRESS: true

.github/workflows/releaseDoc.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         #        runs-on: [ ubuntu-20.04, macOS-latest]
-        runs-on: [ ubuntu-20.04 ]
+        runs-on: [ ubuntu-22.04 ]
       fail-fast: false
 
     runs-on: ${{ matrix.runs-on }}

.github/workflows/ubuntu20.04.yml → .github/workflows/ubuntu24.04.yml

@@ -1,4 +1,4 @@
-name: Ubuntu 20.04
+name: Ubuntu 24.04
 
 on:
   push:
@@ -7,13 +7,13 @@ on:
 #      - v*
 
 jobs:
-  build_ubuntu_20_04:
-    name:  Build and Test on Ubuntu 20.04
+  build_ubuntu_24_04:
+    name: Build and Test on Ubuntu 24.04
 
     strategy:
       matrix:
-        #        runs-on: [ ubuntu-20.04, macOS-latest]
-        runs-on: [ ubuntu-20.04 ]
+        #        runs-on: [ ubuntu-22.04, macOS-latest]
+        runs-on: [ ubuntu-24.04 ]
       fail-fast: false
 
     runs-on: ${{ matrix.runs-on }}

CHANGELOG.md

@@ -1,47 +1,163 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
-* refactored module widget
-  * added option to show gate content for each module
-  * added option to show interior nets for each module
-  * added `Isolate in new view` feature for nets
-  * added button to expand or collapse all tree items
-  * added delete module action and shortcut
-  * added entries for context menu
-* refactored search bar
-  * changed appearance of search bar to be more intuitive
-  * added menu for extended options - e.g. option to search in selected columns only
-  * added search history
-  * added filter proxy class for trees and tables increasing the search performance
-* refactored layouter module
-  * switched to multithreaded algorithm
-  * boosted performance by using classes with faster memory access
-  * removed layouter code used prior to version 3.1.0 - thus removing the setting option to use that code
-  * added setting option to dump junction layout input data for experts to debug in case of layout errors
-* module pins
-  * added qualifier for `pin_changed` core events telling receiver details about the recent modification
-  * added event scope and stacking classes so that `pin_changed` events can be collected and prioritized
-  * added specific GUI handler for every `pin_changed` event thus replacing the reload-entire-pingroup-tree policy
-  * added class `ActionPingroup` so that UNDO function works for all pin / pin group actions issued from GUI
+## [Unreleased] 
+
+## [4.4.1](v4.4.1) - 2024-07-29 14:21:42+02:00 (urgency: medium)
+* fixed `hal_py.GateLibrary.gate_types` pybind
+* fixed `hal_py.NetlistFactory.load_netlist` pybind
+
+## [4.4.0](v4.4.0) - 2024-07-19 15:55:24+02:00 (urgency: medium)
+* **WARNING:** this release breaks the API of the `boolean_influence` and `bitorder_propagation` plugin
+* **WARNING:** this release contains many new unstable plugin APIs that will likely change in the future
+* plugins
+  * updated `boolean_influence` plugin
+    * changed API so that no instance of the plugin needs to be created anymore to apply its algorithms
+    * file structure and namespace clean up
+  * added `resynthesis` plugin
+    * moved `decompose_gate` and `decompose_gates_of_type` from `netlist_preprocessing` plugin and slightly changed their API
+    * added functions for logic re-synthesis that write out parts of the current netlist and call Yosys as an open-source synthesizer to produce a new gate-level netlist based on a user-defined gate library
+  * changed `bitorder_propagation` plugin
+    * changed API so that no instance of the plugin needs to be created anymore to apply its algorithms
+    * changed propagation logic for better results
+  * updated `z3_utils`plugin
+    * general code and file structure clean up as well as more documentation
+    * added comprehensive simplification logic that is able to simplify `z3::expr` using an extended rule set as the simplification of `hal::BooleanFunction`
+  * added `module_identification` plugin 
+    * allows a user to automatically search for arithmetic structures in the netlist
+  * added `sequential_symbolic_execution` plugin
+    * allows the user to perform symbolic execution over mulitple cycles including sequential elements
+  * added first rudimentary version of `genlib_writer` that allows to write the combinational gates of a gate library in genlib format which is required for resynthesis.
+* core
+  * decorators
+    * added `NetlistModificationDecorator::add_vcc/gnd_nets()` to create ground and power nets for netlists that do not have a ground/power net already
+  * added `SMT::Solver::query_local()` variant that directly takes an SMT representation of a Solver query
+  * added `Netlist::get_gnd/vcc_nets()` to get all global ground and power nets
+* deps
+    * added `json.hpp` from nlohmann to deps to offer a light weight json api
+    * adapted cmake to consider the correct flags when finding and linking against the new version of Bitwuzla
+* miscellaneous
+  * added logic evaluator plugin
+  * added backward compatibility for view management
+  * slightly improved symbolic execution engine
+  * added a version of `netlist_factory::load_netlist` that takes a path to a netlist file as well as a pointer to a gate library
+  * added `use_net_variables` parameter to `Gate::get_resolved_boolean_function` to choose whether to use input pins or nets as variable names
+  * added `utils::get_unique_temp_directory`
+  * added `base` parameter to `utils::wrapped_stoull` and `utils::wrapped_stoul`
+  * added `all_global_io` parameter to `SubgraphNetlistDecorator::copy_subgraph_netlist` to configure labeling of global inputs and outputs
+  * added datatype `ExistingFile` to plugin parameter 
+  * added helper gate libraries needed for resynthesis; this is a dirty hack, expect more changes later
+  * changed MUX data input and output pins in all gate libraries to `PinType::data`
+* bugfixes
+  * fixed incompatibility between shipped zlib and QuaZip libraries
+  * fixed a bug when checking whether one Boolean function is just a negated version of another one during symbolic execution
+  * fixed bugs related to the Boolean function SLICE operation 
+  * fixed VCD writer of `netlist_simulation_controller` plugin
+  * fixed handling of const `0` and `1` nets in `verilog_parser`, `vhdl_parser`, and `verilog_writer` plugins
+  * fixed layout bug which occured when leftmost node had no inputs
+  * fixed missing sort indicator when sorting entries in `Views` widget
+  * fixed bug loading simulation data by cleaning map before loading controller from project
+  * fixed bug that occured when trying to generate the Boolean influence for a constant Boolean function  
+
+## [4.3.0](v4.3.0) - 2024-07-02 13:42:55+02:00 (urgency: medium)
+* **WARNING:** this release breaks compatibility with Ubuntu 20.04 LTS
+* **WARNING:** this release breaks the API of the `graph_algorithm`, `dataflow`, and `xilinx_toolbox` plugins
+* GUI
+  * refactored module widget
+    * added option to show gate content for each module
+    * added option to show interior nets for each module
+    * added `Isolate in new view` feature for nets
+    * added button to expand or collapse all tree items
+    * added delete module action and shortcut
+    * added entries for context menu
+    * adapted appearance for menu content tree, selection details tree, grouping content tree (same model for all)
+  * refactored view widget
+    * changed appearance from tabular view to tree view
+    * added 'directory' elements to organize and manage groups of views
+    * added drag'n drop feature allowing to relocate views or directories to another branch in the tree
+    * added column for view ID
+    * added functions to Python GUI API to create, modify and delete views and directories
+    * added UNDO functionality for create/delete view and directory actions
+    * fixed sort-by-column feature. The tree is not sorted at program start thus showing elements in 'natural' order.
+  * refactored search bar
+    * changed appearance of search bar to be more intuitive
+    * added menu for extended options - e.g. option to search in selected columns only
+    * added search history
+    * added filter proxy class for trees and tables increasing the search performance
+  * refactored layouter module
+    * switched to multithreaded algorithm
+    * boosted performance by using classes with faster memory access
+    * removed layouter code used prior to version 3.1.0 - thus removing the setting option to use that code
+    * added setting option to dump junction layout input data for experts to debug in case of layout errors
+* plugins
+  * added `hawkeye` plugin for the detection of symmetric cryptographic implementations in gate-level netlists
+    * see publication `HAWKEYE - Recovering Symmetric Cryptography From Hardware Circuits` at CRYPTO'24 for details
+  * changed `graph_algorithm` plugin
+    * updated the igraph dependency shipped with HAL
+    * graph corresponding to a netlist is now encapsulated within an `NetlistGraph` object that allows easy interaction with the graph
+    * added new functions for computing neighborhoods, shortest paths, subgraphs, and (strongly) connected components
+    * changed the API to facilitate for the aforementioned changes and made everything accessible via Python
+  * changed `dataflow_analysis` plugin
+    * can now operate on arbitrary, user-defined gate types, not only FFs 
+    * user can now specify the pin types to be considered as control pins
+    * can now take known registers and other known word-level structures into account during analysis
+    * changed the API to facilitate for the aforementioned changes
+  * changed `netlist_simulator_controller` plugin
+    * added feature to VCD parser: removal of leading backslash and trailing whitespace from waveform name
+    * added converter for net names which don't qualify as C++ variable name
+    * extended maximum line with the CSV parser can handle
+    * changed warning messages for waveform parsing and made them more specific
+    * changed policy toward 'dangling' wires, they are no longer ignored but considered as global inputs or outputs
+  * changed `waveform_viewer` plugin
+    * added GUI wizard to structure input steps when launching a new simulation
+    * added table widget to enter engine parameter
+    * added table widget to enter simulation input data
+    * added viewer to show output of simulation process while running
+    * added algorithm to identify simulated pin groups and bundle apropriate waveform to groups
+  * changed `xilinx_toolbox` plugin
+    * added `split_shift_registers` function to split `SRL16E` gates into multiple flip-flops
+    * changed Python bindings for better usability
+* core
+  * pin (groups)
+    * added optional flag to determine whether a pin group has an inherent order (defaults to `false`)
+    * added `GateType::delete_pin_group` and `GateType::assign_pin_to_group` to enable more operations on pin groups of gate pins
+    * added parameter `force_name` to enforce pin (group) renaming to `Module::set_pin_name`, `Module::set_pin_group_name`, `Module::create_pin`, and `Module::create_pin_group`
+    * added pin types `status`, `error`, `error_detection`, `done`, and `control`
+    * added qualifier for module `pin_changed` core events telling receiver details about the recent modification
+    * added event scope and stacking classes so that module `pin_changed` events can be collected and prioritized
+    * added specific GUI handler for every module `pin_changed` event thus replacing the reload-entire-pingroup-tree policy
+    * added class `ActionPingroup` so that UNDO function works for all pin / pin group actions issued from GUI
+  * decorators
+    * added `NetlistTraversalDecorator` to ease exploration of a netlist
+      * added `get_next_matching_gates` to get successor/predecessor gates matching a certain condition
+      * added `get_next_matching_gates_until` to get successor/predecessor gates until a certain condition is fulfilled
+      * added `get_next_matching_gates_until_depth` to get successor/predecessor gates up to a certain depth
+      * added `get_next_sequential_gates` and `get_next_sequential_gates_map` to get the next layer of sequential successors/predecessors
+      * added `get_next_combinational_gates` to get all combinational gates until the next non-combinational gates are reached
 * miscellaneous
+  * added support for Ubuntu 24.04 LTS
   * added INIT field declaration to FF-gate-types in example library
   * added drag'n drop feature allowing to move several nodes in graph view at same time
-  * added functions to Python GUI API to create, modifiy and delete views
   * added GUI PluginParameter type `ComboBox` for parameters that can be requested from plugin
   * added GUI PluginParameter types `Module` and `Gated` for parameters that can be requested from plugin
   * added `Show content` button to `Groupings` widget to show content of grouping as a list
   * added flag which Python editor tab is active when serializing project
-  * added `GateType::delete_pin_group` and `GateType::assign_pin_to_group` to enable more operations on pin groups of gate pins
   * added extended gate library picker when importing a netlist
   * added keyboard shortcut for delete-item action from toolbar
-  * added parameter `force_name` to enforce pin (group) renaming to `Module::set_pin_name`, `Module::set_pin_group_name`, `Module::create_pin`, and `Module::create_pin_group`
-  * added Python bindings for `GateLibrary::create_gate_type` (cannot handle `GateTypeComponent` parameter as pybind cannot accept `std::unique_ptr` as parameter)
+  * added gate type properties `fifo` and `shift_register`
+  * added optional filter to `Net::get_num_of_sources` and `Net::get_num_of_destinations`
+  * added function `unify_ff_outputs` to netlist preprocessing plugin
+  * added function `replace_gate_type` to gate library
   * changed supported input file formats for import from hard coded list to list provided by loadable parser plugins
   * changed behavior of import netlist dialog, suggest only non-existing directory names and loop until an acceptable name was entered
   * changed appearance and behavior of import project dialog, make sure existing hal projects don't get overwritten
   * changed installation script policy to install Python packages (omit 'pip install' which would need virtual environment)
+  * removed hard coded path names from CI MacOS workflow script
+  * deprecated many functions in `netlist_utils` as they have been moved somewhere else
 * bugfixes
+  * fixed saleae input data reader which gets linked into external Verilator simulation code
+  * fixed waveform viewer: opening old results will no longer generate the same view twice
+  * fixed waveform viewer: opening old results will by now also update waveform time axis
   * fixed colors in Python Console when switching between color schemes
   * fixed pybind of `Module::get_gates`
   * fixed Python script execution abort button disappearing when switching tabs
@@ -55,8 +171,12 @@ All notable changes to this project will be documented in this file.
   * fixed format string handling of enums in log outputs
   * fixed restoring user assigned module colors from project file
   * fixed no scrollbar shown in `Data` tab of `Selection Details` widget
+  * fixed declaration of FF-gate type in example gate library
+  * fixed error which could cause crashes in do-not-render-layout-until-complex-operation-finished algorithm
+  * fixed wrong placements of nodes in view by XML-macro (might even crash)
   * fixed problems in GUI plugin management caused by addressing plugins by absolute path
   * fixed several bugs related to moving node boxes in GUI by drag'n'drop
+  * fixed several bugs in automated tests, eliminate cases which produce non-deterministic results
 
 ## [4.2.0](v4.2.0) - 2023-05-24 10:02:04-07:00 (urgency: medium)
 * GUI plugin manager