EVM test tool

CHANGELOG.md

@@ -6,13 +6,20 @@ The format is based on [Keep a Changelog],
 and this project adheres to [Semantic Versioning].
 
 
+## [0.2.0] - unreleased
+### Added
+- **evm-test** - the testing tool for EVMC-compatible Ethereum Virtual Machine implementations
+  [#85](https://github.com/ethereum/evmone/pull/85).
+
+
 ## [0.1.0] - 2019-06-19
 ### Added
 - First release of the evmone project.
 - Support for all current EVM revisions up to Petersburg.
 - The [intx 0.2.0](https://github.com/chfast/intx/releases/tag/v0.2.0) library is used for 256-bit precision arithmetic. 
 
 
+[0.2.0]: https://github.com/ethereum/evmone/compare/v0.1.0..master
 [0.1.0]: https://github.com/ethereum/evmone/releases/tag/v0.1.0
 
 [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/

CMakeLists.txt

@@ -54,7 +54,14 @@ if(EVMONE_TESTING)
 endif()
 
 
-install(TARGETS evmone EXPORT evmoneTargets
+# INSTALL
+
+set(install_targets evmone)
+if(TARGET evm-test)
+    list(APPEND install_targets evm-test)
+endif()
+
+install(TARGETS ${install_targets} EXPORT evmoneTargets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})

README.md

@@ -43,6 +43,18 @@ bin/evmone-unittests
 bin/evmone-bench
 ```
 
+### Tools
+
+#### evm-test
+
+The **evm-test** executes a collection of unit tests on 
+any EVMC-compatible Ethereum Virtual Machine implementation.
+The collection of tests comes from the evmone project.
+
+```bash
+evm-test ./evmone.so
+```
+
 ## Maintainer
 
 Paweł Bylica [@chfast]

test/CMakeLists.txt

@@ -11,7 +11,7 @@ add_subdirectory(bench)
 add_subdirectory(unittests)
 
 set_target_properties(
-    evmone-bench evmone-unittests testutils PROPERTIES
+    evm-test evmone-bench evmone-unittests testutils PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}
     LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}
     RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}

test/unittests/CMakeLists.txt

@@ -5,19 +5,36 @@
 hunter_add_package(GTest)
 find_package(GTest CONFIG REQUIRED)
 
-add_executable(evmone-unittests
-    analysis_test.cpp
-    bytecode_test.cpp
+# The evm-unittests library contains generic EVM unit tests for EVMC-compatible VMs.
+add_library(evm-unittests OBJECT
     evm_calls_test.cpp
     evm_fixture.hpp
     evm_test.cpp
+)
+target_link_libraries(evm-unittests PRIVATE testutils evmc::evmc GTest::gtest)
+
+# The internal evmone unit tests. The generic EVM ones are also built in.
+add_executable(evmone-unittests
+    analysis_test.cpp
+    bytecode_test.cpp
     evmone_test.cpp
     utils_test.cpp
+    vm_loader_evmone.cpp
 )
-
-target_link_libraries(evmone-unittests PRIVATE evmone testutils GTest::gtest GTest::main)
+target_link_libraries(evmone-unittests PRIVATE evm-unittests evmone testutils GTest::gtest GTest::main)
 target_include_directories(evmone-unittests PRIVATE ${evmone_private_include_dir})
 
-set_source_files_properties(evmone_test.cpp PROPERTIES COMPILE_DEFINITIONS PROJECT_VERSION="${PROJECT_VERSION}")
+gtest_discover_tests(evmone-unittests TEST_PREFIX ${PROJECT_NAME}/unittests/)
 
-gtest_add_tests(TARGET evmone-unittests TEST_PREFIX ${PROJECT_NAME}/unittests/)
+# The evm-test tool that contains the all evm-unittests and loads VMs as EVMC modules.
+add_executable(evm-test main.cpp)
+target_link_libraries(evm-test PRIVATE evm-unittests testutils evmc::evmc evmc::loader GTest::gtest)
+
+add_test(NAME ${PROJECT_NAME}/evm-test COMMAND evm-test $<TARGET_FILE:evmone>)
+
+# Provide the project version to selected source files.
+set_source_files_properties(
+    evmone_test.cpp
+    main.cpp
+    PROPERTIES COMPILE_DEFINITIONS PROJECT_VERSION="${PROJECT_VERSION}"
+)

test/unittests/evm_fixture.hpp

@@ -3,8 +3,7 @@
 // Licensed under the Apache License, Version 2.0.
 #pragma once
 
-#include <evmone/evmone.h>
-
+#include "vm_loader.hpp"
 #include <gtest/gtest.h>
 #include <test/utils/host_mock.hpp>
 #include <test/utils/utils.hpp>
@@ -35,13 +34,13 @@
 class evm : public testing::Test, public MockedHost
 {
 protected:
-    evmc::vm vm;
+    evmc::vm& vm;
     evmc_revision rev = EVMC_BYZANTIUM;  // Byzantium by default. TODO: Add alias evmc::revision.
     evmc_message msg = {};               // TODO: Add evmc::message with default constructor.
     evmc::result result{{}};  // TODO: Add default constructor to evmc::result, update code here.
     int64_t gas_used = 0;
 
-    evm() noexcept : vm{evmc_create_evmone()} {}
+    evm() noexcept : vm{get_vm()} {}
 
     /// Wrapper for evmone::execute. The result will be in the .result field.
     void execute(int64_t gas, bytes_view code, std::string_view input_hex = {}) noexcept

test/unittests/main.cpp

@@ -0,0 +1,179 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2019 The evmone Authors.
+// Licensed under the Apache License, Version 2.0.
+
+#include "vm_loader.hpp"
+#include <evmc/loader.h>
+#include <gtest/gtest.h>
+#include <iostream>
+#include <string>
+#include <vector>
+
+/// The loaded EVMC module.
+static evmc::vm evmc_module;
+
+evmc::vm& get_vm() noexcept
+{
+    return evmc_module;
+}
+
+/// Simple and copy&paste distributable CLI parser.
+///
+/// TODO: Originally taken from EVMC and modified here. Copy it back.
+class cli_parser
+{
+public:
+    using preprocessor_fn = void (*)(int*, char**);
+
+    const char* const application_name = nullptr;
+    const char* const application_version = nullptr;
+    const char* const application_description = nullptr;
+
+    std::vector<std::string> arguments_names;
+    std::vector<std::string> arguments;
+
+    preprocessor_fn preprocessor = [](int*, char**) {};
+
+    cli_parser(const char* app_name, const char* app_version, const char* app_description,
+        std::vector<std::string> args_names) noexcept
+      : application_name{app_name},
+        application_version{app_version},
+        application_description{app_description},
+        arguments_names{std::move(args_names)}
+    {
+        arguments.reserve(this->arguments_names.size());
+    }
+
+    /// Sets the preprocessor and enables preprocessing.
+    ///
+    /// The preprocessor runs on provided arguments before the parsing is done.
+    /// It is allowed to modify the arguments and/or generate other output.
+    ///
+    /// @param fn The preprocessor function.
+    void set_preprocessor(preprocessor_fn fn) noexcept { preprocessor = fn; }
+
+    /// Parses the command line arguments.
+    ///
+    /// It recognize --help and --version built-in options and output for these is sent
+    /// to the @p out output stream.
+    /// Errors are sent to the @p err output stream.
+    ///
+    /// @return Negative value in case of error,
+    ///         0 in case --help or --version was provided and the program should terminate,
+    ///         positive value in case the program should continue.
+    int parse(int argc, char* argv[], std::ostream& out, std::ostream& err)
+    {
+        out << application_name << " " << application_version << "\n\n";
+
+        const auto should_exit = handle_builtin_options(argc, argv, out);
+
+        // Run preprocessor after the output from built-in options.
+        preprocessor(&argc, argv);
+
+        if (should_exit)
+            return 0;
+
+        size_t num_args = 0;
+        for (int i = 1; i < argc; ++i)
+        {
+            auto arg = std::string{argv[i]};
+
+            const auto num_dashes = arg.find_first_not_of('-');
+            if (num_dashes == 0)  // Argument.
+            {
+                ++num_args;
+                if (num_args > arguments_names.size())
+                {
+                    err << "Unexpected argument \"" << arg << "\"\n";
+                    return -1;
+                }
+                arguments.emplace_back(std::move(arg));
+                continue;
+            }
+
+            err << "Unknown option \"" << argv[i] << "\"\n";
+            return -1;
+        }
+
+        if (num_args < arguments_names.size())
+        {
+            for (auto i = num_args; i < arguments_names.size(); ++i)
+                err << "The " << arguments_names[i] << " argument is required.\n";
+            err << "Run with --help for more information.\n";
+            return -1;
+        }
+
+        return 1;
+    }
+
+private:
+    bool handle_builtin_options(int argc, char* argv[], std::ostream& out)
+    {
+        using namespace std::string_literals;
+
+        auto help = false;
+        auto version = false;
+
+        for (int i = 1; i < argc; ++i)
+        {
+            help |= argv[i] == "--help"s || argv[i] == "-h"s;
+            version |= argv[i] == "--version"s;
+        }
+
+        if (help)
+        {
+            out << "Usage: " << argv[0];
+            for (const auto& name : arguments_names)
+                out << " " << name;
+            out << "\n\n";
+            return true;
+        }
+
+        if (version)
+        {
+            if (application_description)
+                out << application_description << "\n";
+            return true;
+        }
+
+        return false;
+    }
+};
+
+int main(int argc, char* argv[])
+{
+    try
+    {
+        auto cli = cli_parser{"EVM Test", PROJECT_VERSION,
+            "Testing tool for EVMC-compatible Ethereum Virtual Machine implementations.\n"
+            "Powered by the evmone project.\n\n"
+            "EVMC:   https://github.com/ethereum/evmc\n"
+            "evmone: https://github.com/ethereum/evmone",
+            {"MODULE"}};
+        cli.set_preprocessor(testing::InitGoogleTest);
+
+        if (const auto error_code = cli.parse(argc, argv, std::cout, std::cerr); error_code <= 0)
+            return error_code;
+
+        const auto& evmc_config = cli.arguments[0];
+        evmc_loader_error_code ec;
+        evmc_module = evmc::vm{evmc_load_and_configure(evmc_config.c_str(), &ec)};
+
+        if (ec != EVMC_LOADER_SUCCESS)
+        {
+            if (const auto error = evmc_last_error_msg())
+                std::cerr << "EVMC loading error: " << error << "\n";
+            else
+                std::cerr << "EVMC loading error " << ec << "\n";
+            return static_cast<int>(ec);
+        }
+
+        std::cout << "Testing " << evmc_config << "\n\n";
+        return RUN_ALL_TESTS();
+    }
+    catch (const std::exception& ex)
+    {
+        std::cerr << ex.what() << "\n";
+        return -2;
+    }
+}

test/unittests/vm_loader.hpp

@@ -0,0 +1,8 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2019 The evmone Authors.
+// Licensed under the Apache License, Version 2.0.
+#pragma once
+
+#include <evmc/evmc.hpp>
+
+evmc::vm& get_vm() noexcept;

test/unittests/vm_loader_evmone.cpp

@@ -0,0 +1,12 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2019 The evmone Authors.
+// Licensed under the Apache License, Version 2.0.
+
+#include "vm_loader.hpp"
+#include <evmone/evmone.h>
+
+evmc::vm& get_vm() noexcept
+{
+    static auto vm = evmc::vm{evmc_create_evmone()};
+    return vm;
+}

test/utils/CMakeLists.txt

@@ -4,11 +4,14 @@
 
 add_library(testutils STATIC
     bytecode.hpp
-    dump.cpp
     host_mock.hpp
     utils.cpp
     utils.hpp
 )
 
-target_link_libraries(testutils PRIVATE evmone evmc::instructions)
-target_include_directories(testutils PUBLIC ${PROJECT_SOURCE_DIR} PRIVATE ${evmone_private_include_dir})
+target_link_libraries(testutils PRIVATE evmc::instructions)
+target_include_directories(testutils PUBLIC ${PROJECT_SOURCE_DIR})
+
+add_library(testutils-dump STATIC dump.cpp)
+target_link_libraries(testutils-dump PRIVATE testutils evmone)
+target_include_directories(testutils-dump PRIVATE ${evmone_private_include_dir})