From 686b0e9f73409adc7688fdf5bac3b7b8382d45ba Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Thu, 2 Aug 2018 12:26:21 -0400 Subject: [PATCH 1/6] Extract stable_unique function --- vowpalwabbit/parser_helper.cc | 36 +++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/vowpalwabbit/parser_helper.cc b/vowpalwabbit/parser_helper.cc index 77b139081a8..6a915c5fb9a 100644 --- a/vowpalwabbit/parser_helper.cc +++ b/vowpalwabbit/parser_helper.cc @@ -34,6 +34,30 @@ std::vector opts_to_args(const std::vector +ForwardIterator stable_unique(ForwardIterator begin, ForwardIterator end) +{ + using value_t = decltype(typename std::iterator_traits::value_type()); + + set unique_set; + + auto current_head = begin; + for (auto current_check = begin; current_check != end; current_check++) + { + if (unique_set.find(*current_check) == unique_set.end()) + { + unique_set.insert(*current_check); + *current_head = *current_check; + current_head++; + } + } + + return current_head; +} + // blackbox wrapping of boost program options to ignore duplicate specification of options allowed only ones, but specified multiple times // Behavior: only the first occurence is kept // Strategy: add one argument after each other until we trigger multiple_occurrences exception. Special care has to be taken of arguments to options. @@ -59,16 +83,8 @@ po::variables_map arguments::add_options_skip_duplicates(po::options_description if (it.second.value().type() == typeid(vector)) { auto& values = it.second.as>(); - set unique_set; - auto current_head = values.begin(); - for (auto current_check = values.begin(); current_check != values.end(); current_check++) - if (unique_set.find(*current_check) == unique_set.end()) - { - unique_set.insert(*current_check); - *current_head = *current_check; - current_head++; - } - values.erase(current_head, values.end()); + auto end = stable_unique(values.begin(), values.end()); + values.erase(end, values.end()); } } From e146b36c88547aa2a7cc1e94e452d14967afbffc Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Thu, 2 Aug 2018 13:24:49 -0400 Subject: [PATCH 2/6] Fix unnecesarily complex type --- vowpalwabbit/parser_helper.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vowpalwabbit/parser_helper.cc b/vowpalwabbit/parser_helper.cc index 6a915c5fb9a..de2873e25f0 100644 --- a/vowpalwabbit/parser_helper.cc +++ b/vowpalwabbit/parser_helper.cc @@ -40,7 +40,7 @@ std::vector opts_to_args(const std::vector ForwardIterator stable_unique(ForwardIterator begin, ForwardIterator end) { - using value_t = decltype(typename std::iterator_traits::value_type()); + using value_t = typename std::iterator_traits::value_type; set unique_set; From 1a8835847567f88def9fb68e9149e053f3ab2458 Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Fri, 3 Aug 2018 15:53:16 -0400 Subject: [PATCH 3/6] Move stable_unique to own header --- vowpalwabbit/parser_helper.cc | 26 +------------------------- vowpalwabbit/stable_unique.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 vowpalwabbit/stable_unique.h diff --git a/vowpalwabbit/parser_helper.cc b/vowpalwabbit/parser_helper.cc index de2873e25f0..04210cd0e08 100644 --- a/vowpalwabbit/parser_helper.cc +++ b/vowpalwabbit/parser_helper.cc @@ -1,7 +1,7 @@ #include #include "parser_helper.h" #include -#include +#include "stable_unique.h" using namespace std; @@ -34,30 +34,6 @@ std::vector opts_to_args(const std::vector -ForwardIterator stable_unique(ForwardIterator begin, ForwardIterator end) -{ - using value_t = typename std::iterator_traits::value_type; - - set unique_set; - - auto current_head = begin; - for (auto current_check = begin; current_check != end; current_check++) - { - if (unique_set.find(*current_check) == unique_set.end()) - { - unique_set.insert(*current_check); - *current_head = *current_check; - current_head++; - } - } - - return current_head; -} - // blackbox wrapping of boost program options to ignore duplicate specification of options allowed only ones, but specified multiple times // Behavior: only the first occurence is kept // Strategy: add one argument after each other until we trigger multiple_occurrences exception. Special care has to be taken of arguments to options. diff --git a/vowpalwabbit/stable_unique.h b/vowpalwabbit/stable_unique.h new file mode 100644 index 00000000000..c4de0a29adc --- /dev/null +++ b/vowpalwabbit/stable_unique.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +// Performs unique operation over collection without requiring the collection to be sorted first. +// Returns pointer to first element after unique elements to erase from until end of collection. +// Original ordering of elements is preserved. +template +ForwardIterator stable_unique(ForwardIterator begin, ForwardIterator end) +{ + using value_t = typename std::iterator_traits::value_type; + + std::set unique_set; + + auto current_head = begin; + for (auto current_check = begin; current_check != end; current_check++) + { + if (unique_set.find(*current_check) == unique_set.end()) + { + unique_set.insert(*current_check); + *current_head = *current_check; + current_head++; + } + } + + return current_head; +} \ No newline at end of file From 876c65f4c036bd8361e78a0689c3435c955bc58b Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Tue, 21 Aug 2018 14:42:09 -0400 Subject: [PATCH 4/6] Add stable_unique test --- Makefile | 6 ++++- test/unit_test/Makefile | 33 +++++++++++++++++++++++++++ test/unit_test/main.cc | 3 +++ test/unit_test/stable_unique_tests.cc | 29 +++++++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 test/unit_test/Makefile create mode 100644 test/unit_test/main.cc create mode 100644 test/unit_test/stable_unique_tests.cc diff --git a/Makefile b/Makefile index ea7045090a3..c0246e968b5 100644 --- a/Makefile +++ b/Makefile @@ -151,12 +151,16 @@ java: vw .FORCE: -test: .FORCE vw library_example +test: .FORCE vw library_example unit_test @echo "vw running test-suite..." (cd test && ./RunTests -d -fe -E 0.001 -O --onethread ../vowpalwabbit/vw) (cd test && ./RunTests -d -fe -E 0.001 ../vowpalwabbit/vw) cd test && python save_resume_test.py --verbose_on_fail +unit_test: vw + cd test/unit_test; $(MAKE) -j $(NPROCS) things + (cd test/unit_test && ./vw-unit-test.out) + test_gcov: .FORCE vw_gcov library_example_gcov @echo "vw running test-suite..." (cd test && ./RunTests -d -fe -E 0.001 ../vowpalwabbit/vw ../vowpalwabbit/vw) diff --git a/test/unit_test/Makefile b/test/unit_test/Makefile new file mode 100644 index 00000000000..6dccac8f3de --- /dev/null +++ b/test/unit_test/Makefile @@ -0,0 +1,33 @@ +TARGET = vw-unit-test.out + +VW_LIB = -L../../vowpalwabbit -lvw -lallreduce +BOOST_LIBS = -lboost_unit_test_framework -lboost_system +ALL_LIBS = $(VW_LIB) $(BOOST_LIBS) $(LIBS) + +INCLUDE = -I ../../vowpalwabbit -I ../../explore + +.PHONY: default all clean + +default: $(TARGET) +all: default + +things: all + +SOURCES = $(wildcard *.cc) +OBJECTS = $(patsubst %.cc, %.o, $(SOURCES)) +HEADERS = $(wildcard *.h) + +%.o: %.cc $(HEADERS) + $(CXX) $(FLAGS) $(INCLUDE) -c $< -o $@ + +.PRECIOUS: $(TARGET) $(OBJECTS) + +$(info $$SOURCES is [${SOURCES}]) +$(info $$OBJECTS is [${OBJECTS}]) + +$(TARGET): $(OBJECTS) + $(CXX) $(FLAGS) $(OBJECTS) $(LIBDIR) $(ALL_LIBS) -Wall -o $@ + +clean: + -rm -f *.o + -rm -f $(TARGET) diff --git a/test/unit_test/main.cc b/test/unit_test/main.cc new file mode 100644 index 00000000000..95489cab72b --- /dev/null +++ b/test/unit_test/main.cc @@ -0,0 +1,3 @@ +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE Main +#include diff --git a/test/unit_test/stable_unique_tests.cc b/test/unit_test/stable_unique_tests.cc new file mode 100644 index 00000000000..7b7a9ef4e79 --- /dev/null +++ b/test/unit_test/stable_unique_tests.cc @@ -0,0 +1,29 @@ +#define BOOST_TEST_DYN_LINK + +#include +#include + +#include +#include + +#include "stable_unique.h" + +BOOST_AUTO_TEST_CASE(stable_unique_sorted) +{ + std::vector v = {1,1,2,2,2,6,78,89,89}; + std::vector expected = {1,2,6,78,89}; + + v.erase(stable_unique(v.begin(), v.end()), v.end()); + + BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end()); +} + +BOOST_AUTO_TEST_CASE(stable_unique_unsorted) +{ + std::vector v = {45,65,76,23,45,11,34,23,45,-34,-1,22,23,22,0,0,11,23,109}; + std::vector expected = {45,65,76,23,11,34,-34,-1,22,0,109}; + + v.erase(stable_unique(v.begin(), v.end()), v.end()); + + BOOST_CHECK_EQUAL_COLLECTIONS(v.begin(), v.end(), expected.begin(), expected.end()); +} From 97ef42708698356d4e3c32409d88296ff24c04f7 Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Tue, 21 Aug 2018 15:45:29 -0400 Subject: [PATCH 5/6] add project to vw.sln for unit tests --- .../unit_test/unit_test.vcxproj | 2 +- test/unit_test/packages.config | 6 + test/unit_test/stable_unique_tests.cc | 1 - test/unit_test/unit_test.vcxproj | 192 ++++++++++++++++++ test/unit_test/unit_test.vcxproj.filters | 28 +++ vowpalwabbit/vw.sln | 18 ++ 6 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 test/unit_test/packages.config create mode 100644 test/unit_test/unit_test.vcxproj create mode 100644 test/unit_test/unit_test.vcxproj.filters diff --git a/reinforcement_learning/unit_test/unit_test.vcxproj b/reinforcement_learning/unit_test/unit_test.vcxproj index e2bee18ea27..56c5e586704 100644 --- a/reinforcement_learning/unit_test/unit_test.vcxproj +++ b/reinforcement_learning/unit_test/unit_test.vcxproj @@ -203,4 +203,4 @@ - \ No newline at end of file + diff --git a/test/unit_test/packages.config b/test/unit_test/packages.config new file mode 100644 index 00000000000..26c6edfbc3b --- /dev/null +++ b/test/unit_test/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/test/unit_test/stable_unique_tests.cc b/test/unit_test/stable_unique_tests.cc index 7b7a9ef4e79..2fc3ece8174 100644 --- a/test/unit_test/stable_unique_tests.cc +++ b/test/unit_test/stable_unique_tests.cc @@ -4,7 +4,6 @@ #include #include -#include #include "stable_unique.h" diff --git a/test/unit_test/unit_test.vcxproj b/test/unit_test/unit_test.vcxproj new file mode 100644 index 00000000000..212ed078ae5 --- /dev/null +++ b/test/unit_test/unit_test.vcxproj @@ -0,0 +1,192 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + {1e205806-7f80-47dd-a38d-fc08083f3593} + + + + {E02E3869-D9AD-4513-B352-93F90B7D6FE3} + Win32Proj + unit_test + vw_unit_test + 8.1 + + + + Application + true + v140 + Unicode + + + Application + false + v140 + true + Unicode + + + Application + true + v140 + Unicode + + + Application + false + v140 + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ + + + true + + + false + $(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ + + + false + + + + Use + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + Level3 + Use + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + Level3 + Use + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + + + + $(SolutionDir);%(AdditionalIncludeDirectories) + + + %(AdditionalLibraryDirectories) + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + %(PreprocessorDefinitions) + true + true + true + true + + + \ No newline at end of file diff --git a/test/unit_test/unit_test.vcxproj.filters b/test/unit_test/unit_test.vcxproj.filters new file mode 100644 index 00000000000..8177266bee9 --- /dev/null +++ b/test/unit_test/unit_test.vcxproj.filters @@ -0,0 +1,28 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + + + + \ No newline at end of file diff --git a/vowpalwabbit/vw.sln b/vowpalwabbit/vw.sln index 48eaf480437..96887433034 100644 --- a/vowpalwabbit/vw.sln +++ b/vowpalwabbit/vw.sln @@ -92,6 +92,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "rl_sim_cpp", "..\reinforcem EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_cpp", "..\reinforcement_learning\examples\test_cpp\test_cpp.vcxproj", "{47259EFF-C4C2-4F8D-AE67-1FA6F6D5D9FE}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unit_test", "..\test\unit_test\unit_test.vcxproj", "{E02E3869-D9AD-4513-B352-93F90B7D6FE3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -527,6 +529,22 @@ Global {47259EFF-C4C2-4F8D-AE67-1FA6F6D5D9FE}.Release|x64.Build.0 = Release|x64 {47259EFF-C4C2-4F8D-AE67-1FA6F6D5D9FE}.Release|x86.ActiveCfg = Release|Win32 {47259EFF-C4C2-4F8D-AE67-1FA6F6D5D9FE}.Release|x86.Build.0 = Release|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Debug|Any CPU.ActiveCfg = Debug|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Debug|x64.ActiveCfg = Debug|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Debug|x64.Build.0 = Debug|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Debug|x86.ActiveCfg = Debug|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Debug|x86.Build.0 = Debug|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.DebugLeakCheck|Any CPU.ActiveCfg = Release|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.DebugLeakCheck|Any CPU.Build.0 = Release|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.DebugLeakCheck|x64.ActiveCfg = Debug|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.DebugLeakCheck|x64.Build.0 = Debug|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.DebugLeakCheck|x86.ActiveCfg = Debug|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.DebugLeakCheck|x86.Build.0 = Debug|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Release|Any CPU.ActiveCfg = Release|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Release|x64.ActiveCfg = Release|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Release|x64.Build.0 = Release|x64 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Release|x86.ActiveCfg = Release|Win32 + {E02E3869-D9AD-4513-B352-93F90B7D6FE3}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 214cd01b9dc0e3a781cf450f9e23ed40a5edd458 Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Tue, 21 Aug 2018 21:04:26 -0400 Subject: [PATCH 6/6] Remove precompiled header from all configurations --- test/unit_test/unit_test.vcxproj | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/unit_test/unit_test.vcxproj b/test/unit_test/unit_test.vcxproj index 212ed078ae5..4995ce0da3b 100644 --- a/test/unit_test/unit_test.vcxproj +++ b/test/unit_test/unit_test.vcxproj @@ -98,7 +98,7 @@ - Use + Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) @@ -111,7 +111,8 @@ - NotUsing + + Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) @@ -125,7 +126,7 @@ Level3 - Use + MaxSpeed true true @@ -142,7 +143,7 @@ Level3 - Use + MaxSpeed true true @@ -189,4 +190,4 @@ true - \ No newline at end of file +