Merge pull request #2276 from DARMA-tasking/2275-improve-perftestharn…

…ess-output #2275: Improve `PerfTestHarness`
DARMA-tasking · May 8, 2024 · b16eb7a · b16eb7a
2 parents a4a29a2 + 0b1b04b
commit b16eb7a
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 37 deletions.
diff --git a/tests/perf/common/test_harness.cc b/tests/perf/common/test_harness.cc
@@ -235,6 +235,10 @@ std::string PerfTestHarness::OutputMemoryUse() const {
     auto const node = per_node_mem.first;
     auto const& memory_use = per_node_mem.second;
 
+    if(memory_use.empty()){
+      continue;
+    }
+
     std::size_t cur_min = std::numeric_limits<std::size_t>::max();
     std::size_t cur_max = 0;
 
@@ -317,12 +321,16 @@ void PerfTestHarness::DumpResults() {
     );
 
     auto const time_file_data = OutputTimeResults();
-    auto const memory_file = OutputMemoryUse();
-
     if (gen_file_) {
-      OutputToFile(fmt::format("{}_mem", name_), memory_file);
       OutputToFile(fmt::format("{}_time", name_), time_file_data);
     }
+
+    if(print_memory_use_) {
+      auto const memory_file = OutputMemoryUse();
+      if (gen_file_) {
+        OutputToFile(fmt::format("{}_mem", name_), memory_file);
+      }
+    }
   }
 }
 
@@ -443,6 +451,15 @@ void PerfTestHarness::SpinScheduler() {
 void PerfTestHarness::GetMemoryUsage() {
   // Memory footprint from PerfTestHarness' internal data structs are included
   memory_use_[current_run_].push_back(mem_tracker_.getUsage());
+  print_memory_use_ = true;
+}
+
+void PerfTestHarness::DisableGlobalTimer() {
+  print_total_time_ = false;
+}
+
+bool PerfTestHarness::ShouldOutputGlobalTimer() const {
+  return print_total_time_;
 }
 
 }}}} // namespace vt::tests::perf::common
diff --git a/tests/perf/common/test_harness.h b/tests/perf/common/test_harness.h
@@ -167,13 +167,28 @@ struct PerfTestHarness : TestHarnessBase {
    */
   void GetMemoryUsage();
 
+/**
+ * \brief Disables the global timer (execution time of VT_PERF_TEST).
+ * Useful for tests that use custom timers.
+ */
+void DisableGlobalTimer();
+
+/**
+ * \brief Returns information whether global timer (execution time of VT_PERF_TEST)
+ * is disabled.
+ */
+bool ShouldOutputGlobalTimer() const;
+
+
 private:
   std::string OutputMemoryUse() const;
   std::string OutputTimeResults();
 
 protected:
   bool gen_file_ = false;
   bool verbose_ = false;
+  bool print_memory_use_ = false;
+  bool print_total_time_ = true;
   uint32_t num_runs_ = 20;
   uint32_t current_run_ = 0;
   std::vector<char*> custom_args_ = {};

diff --git a/tests/perf/common/test_harness_macros.h b/tests/perf/common/test_harness_macros.h
@@ -106,40 +106,46 @@ struct PerfTestRegistry{
   } StructName##TestName##_registerer;                                     \
   void StructName##TestName::TestFunc()
 
-#define VT_PERF_TEST_MAIN()                                                      \
-  int main(int argc, char** argv) {                                              \
-    using namespace vt::tests::perf::common;                                     \
-    MPI_Init(&argc, &argv);                                                      \
-    int rank;                                                                    \
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);                                        \
-    for (const auto& test_base : PerfTestRegistry::GetTests()) {                 \
-      auto* test = dynamic_cast<PerfTestHarness*>(test_base.get());              \
-      test->Initialize(argc, argv);                                              \
-      auto const num_runs = test->GetNumRuns();                                  \
-      StopWatch timer;                                                           \
-      if (rank == 0) {                                                           \
-        fmt::print("{}RUNNING TEST:{} {} (Number of runs = {}) ...\n",           \
-          vt::debug::bold(), vt::debug::reset(), vt::debug::reg(test->GetName()),\
-          vt::debug::reg(fmt::format("{}", num_runs)));                          \
-      }                                                                          \
-      for (uint32_t run_num = 1; run_num <= num_runs; ++run_num) {               \
-        test->SetUp();                                                           \
-                                                                                 \
-        timer.Start();                                                           \
-        test->TestFunc();                                                        \
-        PerfTestHarness::SpinScheduler();                                        \
-        test->AddResult({test->GetName(), timer.Stop()});                        \
-                                                                                 \
-        if (run_num == num_runs) {                                               \
-          test->SyncResults();                                                   \
-        }                                                                        \
-                                                                                 \
-        test->TearDown();                                                        \
-      }                                                                          \
-      test->DumpResults();                                                       \
-    }                                                                            \
-    MPI_Finalize();                                                              \
-    return 0;                                                                    \
+#define VT_PERF_TEST_MAIN()                                                  \
+  int main(int argc, char** argv) {                                          \
+    using namespace vt::tests::perf::common;                                 \
+    MPI_Init(&argc, &argv);                                                  \
+    int rank;                                                                \
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);                                    \
+    auto& tests = PerfTestRegistry::GetTests();                              \
+    for (uint32_t test_num = 0; test_num < tests.size(); ++test_num) {       \
+      auto* test = dynamic_cast<PerfTestHarness*>(tests.at(test_num).get()); \
+      test->Initialize(argc, argv);                                          \
+      auto const num_runs = test->GetNumRuns();                              \
+      StopWatch timer;                                                       \
+      if (rank == 0) {                                                       \
+        fmt::print(                                                          \
+          "{}{}RUNNING TEST:{} {} (Number of runs = {}) ...\n",              \
+          test_num > 0 ? "\n\n\n\n" : "", vt::debug::bold(),                 \
+          vt::debug::reset(), vt::debug::reg(test->GetName()),               \
+          vt::debug::reg(fmt::format("{}", num_runs)));                      \
+      }                                                                      \
+      for (uint32_t run_num = 1; run_num <= num_runs; ++run_num) {           \
+        test->SetUp();                                                       \
+                                                                             \
+        timer.Start();                                                       \
+        test->TestFunc();                                                    \
+        PerfTestHarness::SpinScheduler();                                    \
+                                                                             \
+        if (test->ShouldOutputGlobalTimer()) {                               \
+          test->AddResult({test->GetName(), timer.Stop()});                  \
+        }                                                                    \
+                                                                             \
+        if (run_num == num_runs) {                                           \
+          test->SyncResults();                                               \
+        }                                                                    \
+                                                                             \
+        test->TearDown();                                                    \
+      }                                                                      \
+      test->DumpResults();                                                   \
+    }                                                                        \
+    MPI_Finalize();                                                          \
+    return 0;                                                                \
   }
 
 }}}} // namespace vt::tests::perf::common