testeth print test suite suggestions

CHANGELOG.md

@@ -9,6 +9,7 @@
 - Added: [#5575](https://github.com/ethereum/aleth/pull/5575) Log active peer count and peer list every 30 seconds.
 - Added: [#5580](https://github.com/ethereum/aleth/pull/5580) Enable syncing from ETC nodes for blocks < dao hard fork block.
 - Added: [#5591](https://github.com/ethereum/aleth/pull/5591) Network logging bugfixes and improvements and add p2pcap log channel.
+- Added: [#5588](https://github.com/ethereum/aleth/pull/5588) Testeth prints similar test suite name suggestions, when the name passed in `-t` argument is not found.
 - Added: [#5593](https://github.com/ethereum/aleth/pull/5593) Dynamically updating host ENR.
 - Changed: [#5559](https://github.com/ethereum/aleth/pull/5559) Update peer validation error messages.
 - Changed: [#5568](https://github.com/ethereum/aleth/pull/5568) Improve rlpx handshake log messages and create new rlpx log channel.

test/CMakeLists.txt

@@ -54,6 +54,7 @@ list(REMOVE_ITEM sources ${unittest_sources})
 # search for test names and create ctest tests
 set(excludeSuites jsonrpc \"customTestSuite\" BlockQueueSuite)
 set(allSuites jsonrpc)
+set(allTests "")
 foreach(file ${sources})
     file(STRINGS ${file} test_list_raw REGEX "BOOST_.*TEST_(SUITE|CASE|SUITE_END)")
     set(TestSuite "DEFAULT")
@@ -82,6 +83,7 @@ foreach(file ${sources})
                     set(TestEthArgs -t ${TestSuitePathFixed} -- ${TESTETH_ARGS})
                     add_test(NAME ${TestSuitePathFixed} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND testeth ${TestEthArgs})
                     set_tests_properties(${TestSuitePathFixed} PROPERTIES TIMEOUT ${timeout})
+                    set(allTests "${allTests}   \"${TestSuitePathFixed}\",\n")
                 endif()
             endif()
         elseif(test MATCHES "^CASE .*")
@@ -93,6 +95,11 @@ foreach(file ${sources})
                     set(TestEthArgs -t ${TestSuitePathFixed}/${TestCase} -- ${TESTETH_ARGS})
                     add_test(NAME ${TestSuitePathFixed}/${TestCase} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND testeth ${TestEthArgs})
                     set_tests_properties(${TestSuitePathFixed}/${TestCase} PROPERTIES TIMEOUT ${timeout})
+                    set(allTests "${allTests}   \"${TestSuitePathFixed}/${TestCase}\",\n")
+                    if (NOT ";${allSuites};" MATCHES ";${TestSuitePathFixed};")
+                        list(APPEND allSuites ${TestSuitePathFixed})
+                        set(allTests "${allTests}   \"${TestSuitePathFixed}\",\n")
+                    endif()
                 endif()
             endif()
         elseif (";${test_raw};" MATCHES "BOOST_AUTO_TEST_SUITE_END()")
@@ -103,6 +110,9 @@ foreach(file ${sources})
     endforeach(test_raw)
 endforeach(file)
 
+#Save allTests into a variable in include file
+configure_file(tools/libtesteth/AllTestNames.h.in ${PROJECT_BINARY_DIR}/AllTestNames.h)
+
 hunter_add_package(yaml-cpp)
 find_package(yaml-cpp CONFIG REQUIRED)
 

test/tools/libtesteth/AllTestNames.h.in

@@ -0,0 +1,8 @@
+#pragma once
+#include <string>
+#include <vector>
+
+std::vector<std::string> const c_allTestNames 
+{
+@allTests@
+};

test/tools/libtesteth/TestHelper.cpp

@@ -637,6 +637,57 @@ string prepareLLLCVersionString()
     return "Error getting LLLC Version";
 }
 
+// A simple C++ implementation of the Levenshtein distance algorithm to measure the amount of
+// difference between two strings. https://gist.github.com/TheRayTracer/2644387
+size_t levenshteinDistance(char const* _s, size_t _n, char const* _t, size_t _m)
+{
+    ++_n;
+    ++_m;
+    size_t* d = new size_t[_n * _m];
+
+    memset(d, 0, sizeof(size_t) * _n * _m);
+    for (size_t i = 1, im = 0; i < _m; ++i, ++im)
+    {
+        for (size_t j = 1, jn = 0; j < _n; ++j, ++jn)
+        {
+            if (_s[jn] == _t[im])
+                d[(i * _n) + j] = d[((i - 1) * _n) + (j - 1)];
+            else
+            {
+                d[(i * _n) + j] = min(d[(i - 1) * _n + j] + 1, /* A deletion. */
+                    min(d[i * _n + (j - 1)] + 1,               /* An insertion. */
+                        d[(i - 1) * _n + (j - 1)] + 1));       /* A substitution. */
+            }
+        }
+    }
+
+    size_t r = d[_n * _m - 1];
+    delete[] d;
+    return r;
+}
+
+std::vector<std::string> testSuggestions(
+    vector<string> const& _testList, std::string const& _sMinusTArg)
+{
+    vector<string> ret;
+    size_t allTestsElementIndex = 0;
+    // <index in availableTests, compared distance>
+    typedef std::pair<size_t, size_t> NameDistance;
+    // Use `vector` here because `set` does not work with sort
+    std::vector<NameDistance> distanceMap;
+    for (auto& it : _testList)
+    {
+        int const dist = test::levenshteinDistance(
+            _sMinusTArg.c_str(), _sMinusTArg.size(), it.c_str(), it.size());
+        distanceMap.emplace_back(allTestsElementIndex++, dist);
+    }
+    std::sort(distanceMap.begin(), distanceMap.end(),
+        [](NameDistance const& _a, NameDistance const& _b) { return _a.second < _b.second; });
+    for (size_t i = 0; i < 3 && i < distanceMap.size(); i++)
+        ret.push_back(_testList[distanceMap[i].first]);
+    return ret;
+}
+
 void copyFile(fs::path const& _source, fs::path const& _destination)
 {
     fs::ifstream src(_source, ios::binary);

test/tools/libtesteth/TestHelper.h

@@ -105,6 +105,7 @@ bytes importByteArray(std::string const& _str);
 void requireJsonFields(json_spirit::mObject const& _o, std::string const& _section,
     std::map<std::string, json_spirit::Value_type> const& _validationMap);
 void checkHexHasEvenLength(std::string const&);
+size_t levenshteinDistance(char const* _s, size_t _n, char const* _t, size_t _m);
 void copyFile(boost::filesystem::path const& _source, boost::filesystem::path const& _destination);
 eth::LogEntries importLog(json_spirit::mArray const& _o);
 std::string exportLog(eth::LogEntries const& _logs);
@@ -131,6 +132,8 @@ json_spirit::mObject fillJsonWithStateChange(eth::State const& _stateOrig, eth::
 json_spirit::mObject fillJsonWithState(eth::State const& _state);
 json_spirit::mObject fillJsonWithState(eth::State const& _state, eth::AccountMaskMap const& _map);
 json_spirit::mObject fillJsonWithTransaction(eth::Transaction const& _txn);
+std::vector<std::string> testSuggestions(
+    std::vector<std::string> const& _testList, std::string const& _sMinusTArg);
 
 //Fill Test Functions
 bool createRandomTest();	//returns true if succeed, false if there was an error;

test/tools/libtesteth/boostTest.cpp

@@ -11,6 +11,7 @@
 #define BOOST_TEST_MODULE EthereumTests
 #define BOOST_TEST_NO_MAIN
 
+#include <AllTestNames.h>
 #include <test/tools/jsontests/BlockChainTests.h>
 #include <test/tools/jsontests/StateTests.h>
 #include <test/tools/jsontests/TransactionTests.h>
@@ -27,6 +28,7 @@ using namespace boost::unit_test;
 static std::ostringstream strCout;
 std::streambuf* oldCoutStreamBuf;
 std::streambuf* oldCerrStreamBuf;
+void printTestSuiteSuggestions(string const& _sMinusTArg);
 
 void customTestSuite()
 {
@@ -133,10 +135,34 @@ int main(int argc, const char* argv[])
         framework::master_test_suite().add(ts1);
     }
 
+    string sMinusTArg;
+    // unit_test_main delete this option from _argv
+    for (int i = 0; i < argc; i++)  // find -t boost arg
+    {
+        std::string const arg = std::string{argv[i]};
+        if (arg == "-t" && i + 1 < argc)
+        {
+            sMinusTArg = std::string{argv[i + 1]};
+            break;
+        }
+    }
+
     std::cout << "Running tests using path: " << test::getTestPath() << std::endl;
     int result = 0;
     auto fakeInit = [](int, char* []) -> boost::unit_test::test_suite* { return nullptr; };
+
     result = unit_test_main(fakeInit, argc, const_cast<char**>(argv));
+    // Print suggestions of a test case if test suite not found
+    if (result == boost::exit_exception_failure && !dev::test::inArray(c_allTestNames, sMinusTArg))
+        printTestSuiteSuggestions(sMinusTArg);
     dev::test::TestOutputHelper::get().printTestExecStats();
     return result;
 }
+
+void printTestSuiteSuggestions(string const& _sMinusTArg)
+{
+    auto const testList = test::testSuggestions(c_allTestNames, _sMinusTArg);
+    std::cerr << "Did you mean: \n";
+    for (auto const& element : testList)
+        std::cerr << "-t " << element << "\n";
+}

test/unittests/libtesteth/testHelperTest.cpp

@@ -28,6 +28,7 @@ using namespace dev::test;
 
 BOOST_FIXTURE_TEST_SUITE(TestHelperSuite, TestOutputHelperFixture)
 
+BOOST_AUTO_TEST_SUITE(TranslateNetworks)
 BOOST_AUTO_TEST_CASE(translateNetworks_gtConstantinople)
 {
     set<string> networks = {">Constantinople"};
@@ -104,5 +105,98 @@ BOOST_AUTO_TEST_CASE(translateNetworks_leFrontier)
             BOOST_REQUIRE(networks.count(test::netIdToString(net)) == 0);
     }
 }
+BOOST_AUTO_TEST_SUITE_END()
+
+BOOST_AUTO_TEST_SUITE(TestHelper)
+BOOST_AUTO_TEST_CASE(levenshteinDistance_similar)
+{
+    char const* word1 = "someword";
+    char const* word2 = "soemword";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 2);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_similar2)
+{
+    char const* word1 = "sOmeWord";
+    char const* word2 = "someword";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 2);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_similar3)
+{
+    char const* word1 = "sOmeWoRd";
+    char const* word2 = "someword";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 3);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_similar4)
+{
+    char const* word1 = "sOmeWoRd";
+    char const* word2 = "soemword";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 5);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_AgtB)
+{
+    char const* word1 = "someword";
+    char const* word2 = "other";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 4);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_AgtB2)
+{
+    char const* word1 = "some long sentence here";
+    char const* word2 = "other shorter phrase";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 14);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_BgtA)
+{
+    char const* word1 = "other";
+    char const* word2 = "someword";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 4);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_BgtA2)
+{
+    char const* word1 = "other shorter phrase";
+    char const* word2 = "some long sentence here";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 14);
+}
+
+BOOST_AUTO_TEST_CASE(levenshteinDistance_different)
+{
+    char const* word1 = "abdefg";
+    char const* word2 = "hijklmn";
+    size_t distance = test::levenshteinDistance(word1, strlen(word1), word2, strlen(word2));
+    BOOST_CHECK_EQUAL(distance, 6);
+}
+
+BOOST_AUTO_TEST_CASE(getTestSuggestions)
+{
+    vector<string> const testList = {
+        "test1", "test2", "BlockSuite", "BlockSuite/TestCase", "GeneralBlockchainTests"};
+    auto list = test::testSuggestions(testList, "blocksuit");
+    BOOST_CHECK(test::inArray(list, string("BlockSuite")));
+}
+
+BOOST_AUTO_TEST_CASE(getTestSuggestions2)
+{
+    vector<string> const testList = {"test1", "test2", "BlockSuite", "BlockSuite/TestCase",
+        "GeneralBlockchainTests", "GeneralStateTests/stExample", "BCGeneralStateTests/stExample"};
+
+    auto list = test::testSuggestions(testList, "GeneralStateTests/stExample2");
+    BOOST_CHECK(test::inArray(list, string("GeneralStateTests/stExample")));
+    BOOST_CHECK(test::inArray(list, string("BCGeneralStateTests/stExample")));
+}
+BOOST_AUTO_TEST_SUITE_END()
 
 BOOST_AUTO_TEST_SUITE_END()