📈 reducing benchmark variance (#365)

nlohmann · Nov 23, 2016 · ed61111 · ed61111
1 parent c34b41a
commit ed61111
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -95,7 +95,7 @@ pretty:
 # benchmarks
 json_benchmarks: benchmarks/benchmarks.cpp benchmarks/benchpress.hpp benchmarks/cxxopts.hpp src/json.hpp
 	cd benchmarks/files/numbers ; python generate.py
-	$(CXX) -std=c++11 $(CXXFLAGS) -DNDEBUG -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
+	$(CXX) -std=c++11 -pthread $(CXXFLAGS) -DNDEBUG -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
 	./json_benchmarks
 
 

diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp
@@ -3,74 +3,114 @@
 #include <fstream>
 #include <benchpress.hpp>
 #include <json.hpp>
+#include <pthread.h>
+#include <thread>
+
+struct StartUp
+{
+    StartUp()
+    {
+#ifndef __llvm__
+        // pin thread to a single CPU
+        cpu_set_t cpuset;
+        pthread_t thread;
+        thread = pthread_self();
+        CPU_ZERO(&cpuset);
+        CPU_SET(std::thread::hardware_concurrency() - 1, &cpuset);
+        pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
+#endif
+    }
+};
+StartUp startup;
 
 BENCHMARK("parse jeopardy.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
 BENCHMARK("parse canada.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/nativejson-benchmark/canada.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
 BENCHMARK("parse citm_catalog.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/nativejson-benchmark/citm_catalog.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
 BENCHMARK("parse twitter.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/nativejson-benchmark/twitter.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
 BENCHMARK("parse numbers/floats.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/numbers/floats.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
 BENCHMARK("parse numbers/signed_ints.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/numbers/signed_ints.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
 BENCHMARK("parse numbers/unsigned_ints.json", [](benchpress::context* ctx)
 {
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->stop_timer();
         std::ifstream input_file("benchmarks/files/numbers/unsigned_ints.json");
         nlohmann::json j;
+        ctx->start_timer();
         j << input_file;
+        ctx->stop_timer();
     }
 })
 
@@ -84,7 +124,9 @@ BENCHMARK("dump jeopardy.json", [](benchpress::context* ctx)
     ctx->reset_timer();
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->start_timer();
         output_file << j;
+        ctx->stop_timer();
     }
 
     std::remove("jeopardy.dump.json");
@@ -100,7 +142,9 @@ BENCHMARK("dump jeopardy.json with indent", [](benchpress::context* ctx)
     ctx->reset_timer();
     for (size_t i = 0; i < ctx->num_iterations(); ++i)
     {
+        ctx->start_timer();
         output_file << std::setw(4) << j;
+        ctx->stop_timer();
     }
 
     std::remove("jeopardy.dump.json");