Known Issue: nvbench breaks loaders. (resolved)

CMakeLists.txt

@@ -34,6 +34,7 @@ set(PROJECT_DEPS_DIR externals)
 # end /* Dependencies directory */
 
 # begin /* Include cmake modules */
+include(${PROJECT_SOURCE_DIR}/cmake/FetchColors.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/FetchThrustCUB.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/FetchModernGPU.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/FetchCXXOpts.cmake)
@@ -58,13 +59,21 @@ set(CMAKE_VERBOSE_MAKEFILE OFF)
 ############ ADD LIBRARY: LOOPS (HEADER-ONLY) ############
 add_library(loops INTERFACE)
 
+
 ####################################################
-############### SET TARGET PROPERTIES ##############
+############### SET SM ARCHITECTURE ################
 ####################################################
-if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-  set(CMAKE_CUDA_ARCHITECTURES 75)
-endif()
 
+## Note: This applies to NVBench as well.
+## Can be used for applications by extracting the
+## CUDA_ARCHITECTURES property from loops project.
+## see: get_target_properties()
+message(STATUS "${Magenta}Set SM Architecture using -DCMAKE_CUDA_ARCHITECTURES=75,80${ColourReset}")
+set(CMAKE_CUDA_ARCHITECTURES 75)
+
+####################################################
+############### SET TARGET PROPERTIES ##############
+####################################################
 set_target_properties(loops 
   PROPERTIES
     CXX_STANDARD 17
@@ -124,7 +133,6 @@ target_link_libraries(loops
 ################# TARGET SOURCES ###################
 ####################################################
 target_sources(loops
-  INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/loops/util/gitsha1make.c"
   INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/loops/container/detail/mmio.cpp"
 )
 

benchmarks/spmv/CMakeLists.txt

@@ -8,7 +8,7 @@ foreach(SOURCE IN LISTS BENCHMARK_SOURCES)
   add_executable(${BENCHMARK_NAME} ${SOURCE})
   target_link_libraries(${BENCHMARK_NAME}
     PRIVATE loops
-    PRIVATE nvbench::main
+    # PRIVATE nvbench::main
     PRIVATE nvbench::nvbench
   )
 

benchmarks/spmv/parameters.hxx

@@ -16,46 +16,73 @@
 #include <cxxopts.hpp>
 
 #include <loops/util/filepath.hxx>
+#include <nvbench/nvbench.cuh>
 
-struct parameters_t {
-  std::string filename;
-  bool help = false;
-  cxxopts::Options options;
+std::string filename;
 
+struct parameters_t {
   /**
    * @brief Construct a new parameters object and parse command line arguments.
    *
    * @param argc Number of command line arguments.
    * @param argv Command line arguments.
    */
-  parameters_t(int argc, char** argv) : options(argv[0], "SPMV Benchmarking") {
-    options.allow_unrecognised_options();
+  parameters_t(int argc, char** argv)
+      : m_options(argv[0], "SPMV Benchmarking"), m_argc(argc) {
+    m_options.allow_unrecognised_options();
     // Add command line options
-    options.add_options()("h,help", "Print help")  // help
+    m_options.add_options()("h,help", "Print help")  // help
         ("m,market", "Matrix file",
          cxxopts::value<std::string>());  // mtx
 
-    // Parse command line arguments
-    auto result = options.parse(argc, argv);
+    // Parse command line arguments.
+    auto result = m_options.parse(argc, argv);
 
+    // Print help if requested
     if (result.count("help")) {
-      help = true;
-      std::cout << options.help({""});
+      m_help = true;
+      std::cout << m_options.help({""});
       std::cout << "  [optional nvbench args]" << std::endl << std::endl;
-      // Do not exit so we also print NVBench help.
-    } else {
+      const char* argh[1] = {"-h"};
+      NVBENCH_MAIN_BODY(1, argh);
+    }
+
+    // Get matrix market file or error if not specified.
+    else {
       if (result.count("market") == 1) {
-        filename = result["market"].as<std::string>();
-        if (!loops::is_market(filename)) {
-          std::cout << options.help({""});
+        this->m_filename = result["market"].as<std::string>();
+        filename = m_filename;
+        if (!loops::is_market(m_filename)) {
+          std::cout << m_options.help({""});
           std::cout << "  [optional nvbench args]" << std::endl << std::endl;
           std::exit(0);
         }
+
+        // Remove loops parameters and pass the rest to nvbench.
+        for (int i = 0; i < argc; i++) {
+          if (strcmp(argv[i], "--market") == 0 || strcmp(argv[i], "-m") == 0) {
+            i++;
+            continue;
+          }
+          m_args.push_back(argv[i]);
+        }
+
       } else {
-        std::cout << options.help({""});
+        std::cout << m_options.help({""});
         std::cout << "  [optional nvbench args]" << std::endl << std::endl;
         std::exit(0);
       }
     }
   }
+
+  /// Helpers for NVBENCH_MAIN_BODY call.
+  int nvbench_argc() { return m_argc - 2; }
+  auto nvbench_argv() { return m_args.data(); }
+
+ private:
+  std::string m_filename;
+  cxxopts::Options m_options;
+  std::vector<const char*> m_args;
+  bool m_help = false;
+  int m_argc;
 };

benchmarks/spmv/work_oriented.cu

@@ -11,15 +11,10 @@
 
 #include "parameters.hxx"
 
-#include <loops/memory.hxx>
-#include <loops/container/vector.hxx>
-#include <loops/util/device.hxx>
-#include <loops/util/sample.hxx>
 #include <loops/util/generate.hxx>
+#include <loops/container/container.hxx>
 #include <loops/algorithms/spmv/work_oriented.cuh>
 
-#include <nvbench/nvbench.cuh>
-
 #define LOOPS_CUPTI_SUPPORTED 0
 
 using namespace loops;
@@ -30,17 +25,14 @@ void work_oriented_bench(nvbench::state& state, nvbench::type_list<value_t>) {
   using offset_t = int;
   using type_t = value_t;
 
-  /// Get sample CSR matrix and create x, y vectors.
-  std::size_t rows = 1 << 20;
-  std::size_t cols = 1 << 20;
-
-  csr_t<index_t, offset_t, value_t> csr;
-  generate::random::csr<index_t, offset_t, type_t>(rows, cols, 0.0000001, csr);
+  matrix_market_t<index_t, offset_t, type_t> mtx;
+  csr_t<index_t, offset_t, type_t> csr(mtx.load(filename));
 
   vector_t<type_t> x(csr.rows);
   vector_t<type_t> y(csr.rows);
 
-  generate::random::uniform_distribution(x.begin(), x.end(), 1, 10);
+  generate::random::uniform_distribution(x.begin(), x.end(), type_t(1.0),
+                                         type_t(10.0));
 
 #if LOOPS_CUPTI_SUPPORTED
   /// Add CUPTI metrics to collect for the state.
@@ -59,4 +51,9 @@ void work_oriented_bench(nvbench::state& state, nvbench::type_list<value_t>) {
 
 // Define a type_list to use for the type axis:
 using value_types = nvbench::type_list<int, float, double>;
-NVBENCH_BENCH_TYPES(work_oriented_bench, NVBENCH_TYPE_AXES(value_types));
+NVBENCH_BENCH_TYPES(work_oriented_bench, NVBENCH_TYPE_AXES(value_types));
+
+int main(int argc, char** argv) {
+  parameters_t params(argc, argv);
+  NVBENCH_MAIN_BODY(params.nvbench_argc(), params.nvbench_argv());
+}

cmake/FetchColors.cmake

@@ -0,0 +1,19 @@
+if(NOT WIN32)
+  string(ASCII 27 Esc)
+  set(ColourReset "${Esc}[m")
+  set(ColourBold  "${Esc}[1m")
+  set(Red         "${Esc}[31m")
+  set(Green       "${Esc}[32m")
+  set(Yellow      "${Esc}[33m")
+  set(Blue        "${Esc}[34m")
+  set(Magenta     "${Esc}[35m")
+  set(Cyan        "${Esc}[36m")
+  set(White       "${Esc}[37m")
+  set(BoldRed     "${Esc}[1;31m")
+  set(BoldGreen   "${Esc}[1;32m")
+  set(BoldYellow  "${Esc}[1;33m")
+  set(BoldBlue    "${Esc}[1;34m")
+  set(BoldMagenta "${Esc}[1;35m")
+  set(BoldCyan    "${Esc}[1;36m")
+  set(BoldWhite   "${Esc}[1;37m")
+endif()

examples/CMakeLists.txt

@@ -2,4 +2,5 @@
 add_subdirectory(range)
 add_subdirectory(saxpy)
 add_subdirectory(spmv)
+add_subdirectory(spmm)
 # end /* Add examples' subdirectories */

examples/spmm/CMakeLists.txt

@@ -0,0 +1,16 @@
+# begin /* Add application */
+set(SOURCES
+  thread_mapped.cu
+)
+
+foreach(SOURCE IN LISTS SOURCES)
+  get_filename_component(TEST_NAME ${SOURCE} NAME_WLE)
+  add_executable(loops.spmm.${TEST_NAME} ${SOURCE})
+  target_link_libraries(loops.spmm.${TEST_NAME} PRIVATE loops)
+  set_target_properties(loops.spmm.${TEST_NAME} 
+    PROPERTIES 
+      CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
+  )
+  message(STATUS "Example Added: loops.spmm.${TEST_NAME}")
+endforeach()
+# end /* Add application */

examples/spmm/helpers.hxx

@@ -0,0 +1,80 @@
+/**
+ * @file helpers.hxx
+ * @author Muhammad Osama (mosama@ucdavis.edu)
+ * @brief Header file for SpMM.
+ * @version 0.1
+ * @date 2022-02-03
+ *
+ * @copyright Copyright (c) 2022
+ *
+ */
+
+#pragma once
+
+#include <loops/util/generate.hxx>
+#include <loops/container/formats.hxx>
+#include <loops/container/vector.hxx>
+#include <loops/container/market.hxx>
+#include <loops/util/filepath.hxx>
+#include <loops/util/equal.hxx>
+#include <loops/util/device.hxx>
+#include <loops/util/timer.hxx>
+#include <loops/memory.hxx>
+#include <cxxopts.hpp>
+
+#include <algorithm>
+#include <iostream>
+
+struct parameters_t {
+  std::string filename;
+  bool validate;
+  bool verbose;
+  cxxopts::Options options;
+
+  /**
+   * @brief Construct a new parameters object and parse command line arguments.
+   *
+   * @param argc Number of command line arguments.
+   * @param argv Command line arguments.
+   */
+  parameters_t(int argc, char** argv)
+      : options(argv[0], "Sparse Matrix-Matrix Multiplication") {
+    // Add command line options
+    options.add_options()("h,help", "Print help")                   // help
+        ("m,market", "Matrix file", cxxopts::value<std::string>())  // mtx
+        ("validate", "CPU validation")                              // validate
+        ("v,verbose", "Verbose output");                            // verbose
+
+    // Parse command line arguments
+    auto result = options.parse(argc, argv);
+
+    if (result.count("help") || (result.count("market") == 0)) {
+      std::cout << options.help({""}) << std::endl;
+      std::exit(0);
+    }
+
+    if (result.count("market") == 1) {
+      filename = result["market"].as<std::string>();
+      if (loops::is_market(filename)) {
+      } else {
+        std::cout << options.help({""}) << std::endl;
+        std::exit(0);
+      }
+    } else {
+      std::cout << options.help({""}) << std::endl;
+      std::exit(0);
+    }
+
+    if (result.count("validate") == 1) {
+      validate = true;
+    } else {
+      validate = false;
+    }
+
+    if (result.count("verbose") == 1) {
+      verbose = true;
+    } else {
+      verbose = false;
+    }
+  }
+};