Rewrite x86 decoder to use Capstone for instruction decoding

CMakeLists.txt

@@ -12,6 +12,15 @@ if (CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR)
     message(FATAL_ERROR "Building in-source is not supported! Create a build dir and remove ${CMAKE_SOURCE_DIR}/CMakeCache.txt")
 endif(CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR)
 
+if (DEFINED ENV{VCPKG_ROOT} AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
+    set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "")
+endif (DEFINED ENV{VCPKG_ROOT} AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
+
+if (CMAKE_TOOLCHAIN_FILE)
+    include("${CMAKE_TOOLCHAIN_FILE}")
+endif (CMAKE_TOOLCHAIN_FILE)
+
+
 set(CMAKE_WARN_DEPRECATED ON)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake-scripts")
 find_package(CCache)

Changelog.md

@@ -1,3 +1,14 @@
+v0.5.0 (in development)
+-----------------------
+- Fixed: Crash when decoding instructions with multiple instruction prefixes in some cases.
+- Feature: The x86 decoder now recognizes a larger subset of the x86 instruction set.
+- Improved: Better high level code output quality for x86 binaries due to more instructions being recognized.
+- Improved: Performance of decoding x86 instructions.
+- Improved: Unit test coverage.
+- Improved: Regression test coverage.
+- Changed: Replaced old pentium (x86) decoder by x86 decoder using libcapstone for decoding instructions.
+- Technical: Improved compilation times and memory usage while compiling.
+
 v0.4.0-alpha (2018-11-11)
 -------------------------
 - Fixed: Crashes when decompiling SPARC binaries.

Readme.md

@@ -19,6 +19,7 @@ it is currently recomended to build the development version (`develop`) of the d
  - A C++ compiler compatible with C++17 (GCC 7+, Clang 5+, MSVC 2017+ are known to work)
  - [CMake 3.8.0 or newer](https://cmake.org/download/)
  - [Qt5](https://www.qt.io/download-open-source/) (Qt 5.9+ is known to work, earlier versions should also work)
+ - [Capstone 3.0.5 or newer](http://www.capstone-engine.org/)
  - [Doxygen 1.8.13 or newer](http://www.doxygen.nl/) (optional, for documentation)
  - [Python 3](https://www.python.org/downloads/) (optional, for regression tests)
 
@@ -29,7 +30,7 @@ On a Linux system you can build and install Boomerang with the usual cmake-make-
 On a Debian-compatible system (e.g. Ubuntu) these commands will clone, build and install Boomerang:
 
 ```bash
-sudo apt-get install git build-essential cmake qt5-default
+sudo apt-get install git build-essential cmake qt5-default libcapstone-dev
 cd YOUR_FAVOURITE_DEVELOPMENT_DIRECTORY
 git clone https://github.com/BoomerangDecompiler/boomerang.git
 cd boomerang && mkdir build && cd build

appveyor.yml

@@ -1,3 +1,11 @@
+#
+# This file is part of the Boomerang Decompiler.
+#
+# See the file "LICENSE.TERMS" for information on usage and
+# redistribution of this file, and for a DISCLAIMER OF ALL
+# WARRANTIES.
+#
+
 version: "Build {build}"
 
 os:
@@ -17,8 +25,7 @@ branches:
 skip_commits:
   message: /\[[aA]ppveyor [sS]kip\]/
 
-pull_requests:
-  do_not_increment_build_number: true
+cache: c:\tools\vcpkg\installed\
 
 clone_folder: C:\projects\boomerang
 
@@ -29,6 +36,9 @@ build:
 matrix:
   fast_finish: true
 
+pull_requests:
+  do_not_increment_build_number: true
+
 init:
   - cmd: cmake --version
   - cmd: msbuild /version
@@ -52,4 +62,3 @@ artifacts:
   - path: 'build\boomerang-*.zip'
     name: Portable ZIP
     type: File
-

cmake-scripts/FindCapstone.cmake

@@ -0,0 +1,90 @@
+#
+# This file is part of the Boomerang Decompiler.
+#
+# See the file "LICENSE.TERMS" for information on usage and
+# redistribution of this file, and for a DISCLAIMER OF ALL
+# WARRANTIES.
+#
+# - Find Capstone disassembler library.
+#
+# This module defines the following variables:
+#   Capstone_FOUND           - true if Capstone was found
+#   Capstone_INCLUDE_DIRS    - Include directories needed for Capstone
+#   Capstone_LIBRARIES       - Libraries to link to when using Capstone
+#   Capstone_DLL             - Path to Capstone DLL, if applicable
+#   Capstone_PDB             - Path to Capstone PDB, if applicable
+
+# Additionally, this module defines the IMPORTED target Capstone::Capstone,
+# if Capstone has been found.
+#
+
+include(FindPackageHandleStandardArgs)
+
+if (Capstone_INCLUDE_DIRS AND Capstone_LIBRARIES)
+    set(Capstone_FIND_QUIETLY TRUE)
+endif (Capstone_INCLUDE_DIRS AND Capstone_LIBRARIES)
+
+find_path(Capstone_INCLUDE_DIR capstone/capstone.h
+    /usr/local/include
+    /usr/include
+    $ENV{MINGDIR}/include
+)
+
+set(Capstone_NAMES capstone capstone_dll)
+find_library(Capstone_LIBRARY
+    NAMES ${Capstone_NAMES}
+    PATHS /usr/local/lib /usr/lib/ $ENV{MINGDIR}/lib
+)
+
+if (WIN32)
+    foreach (DLLNAME ${Capstone_NAMES})
+        find_file(Capstone_DLL
+            NAME ${DLLNAME}.dll
+            PATHS /usr/local/bin /usr/bin/ $ENV{MINGDIR}/bin
+        )
+
+        find_file(Capstone_PDB
+            NAME ${DLLNAME}.pdb
+            PATHS /usr/local/bin /usr/bin/ $ENV{MINGDIR}/bin
+        )
+    endforeach ()
+
+    # Allow dll to be built without debug symbol support
+    find_package_handle_standard_args(Capstone
+        FOUND_VAR Capstone_FOUND
+        REQUIRED_VARS Capstone_LIBRARY Capstone_INCLUDE_DIR Capstone_DLL
+    )
+else (WIN32)
+    find_package_handle_standard_args(Capstone
+        FOUND_VAR Capstone_FOUND
+        REQUIRED_VARS Capstone_LIBRARY Capstone_INCLUDE_DIR
+    )
+endif (WIN32)
+
+
+set(Capstone_INCLUDE_DIRS "${Capstone_INCLUDE_DIR}")
+set(Capstone_LIBRARIES "${Capstone_LIBRARY}")
+
+if (Capstone_FOUND OR NOT Capstone_FIND_REQUIRED)
+    # Only show variables when Capstone is required and not found
+    mark_as_advanced(
+        Capstone_INCLUDE_DIRS
+        Capstone_LIBRARIES
+        Capstone_CONFIG
+        Capstone_INCLUDE_DIR
+        Capstone_LIBRARY
+    )
+
+    if (WIN32)
+        mark_as_advanced(Capstone_DLL Capstone_PDB)
+    endif (WIN32)
+endif (Capstone_FOUND OR NOT Capstone_FIND_REQUIRED)
+
+if (Capstone_FOUND)
+    if (NOT TARGET Capstone::Capstone)
+        add_library(Capstone::Capstone UNKNOWN IMPORTED)
+    endif (NOT TARGET Capstone::Capstone)
+
+    set_target_properties(Capstone::Capstone PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${Capstone_INCLUDE_DIRS}")
+    set_target_properties(Capstone::Capstone PROPERTIES IMPORTED_LOCATION "${Capstone_LIBRARIES}")
+endif (Capstone_FOUND)

cmake-scripts/boomerang-configure.cmake

@@ -98,3 +98,4 @@ add_definitions(-DV9_ONLY=0)
 if (NOT BUILD_SHARED_LIBS)
     add_definitions(-DBOOMERANG_BUILD_STATIC=1)
 endif ()
+

cmake-scripts/boomerang-dependencies.cmake

@@ -6,7 +6,6 @@
 # WARRANTIES.
 #
 
-
 find_package(Qt5Core REQUIRED HINTS $ENV{QTDIR})
 if (Qt5Core_FOUND)
     mark_as_advanced(Qt5Core_DIR)
@@ -18,3 +17,4 @@ if (Qt5Xml_FOUND)
 endif (Qt5Xml_FOUND)
 
 find_package(Threads)
+find_package(Capstone REQUIRED)

data/signatures/common.hs

@@ -90,5 +90,6 @@ signal.h
 netdb.h
 socket.h
 termios.h
+fortran.h
 
 misc.h

data/signatures/fortran.h

@@ -0,0 +1,6 @@
+void f_setarg(int argc, char *argv[]);
+void f_setsig();
+void f_init();
+void f_exit();
+
+void do_lio(void *arg0, void *arg1, void *arg2, int arg3);