merge

.github/workflows/tests.yml

@@ -4,12 +4,12 @@ on: [ push, pull_request ]
 
 jobs:
   kaminpar_tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     continue-on-error: true 
     strategy:
       matrix:
         compiler:
-          #- { name: Clang, cc: clang, cxx: clang++ }
+          - { name: Clang, cc: clang, cxx: clang++ }
           - { name: GNU, cc: gcc, cxx: g++ }
         build-mode: [Release]
     steps:

CMakeLists.txt

@@ -5,7 +5,7 @@ include(FetchContent)
 include(CheckCXXCompilerFlag)
 
 project(KaMinPar
-        DESCRIPTION "Shared-memory and distributed graph partitioner for large k partitioning."
+        DESCRIPTION "Shared-memory and distributed-memory Graph Partitioner"
         LANGUAGES C CXX)
 
 set(PROJECT_VENDOR "Daniel Seemaier")
@@ -34,7 +34,7 @@ option(KAMINPAR_ENABLE_STATISTICS "Generate and output detailed statistics." OFF
 option(KAMINPAR_ENABLE_TIMERS "Measure running times. Must be set to 'OFF' if the library interface is used from multiple threads simulatinously." ON)
 option(KAMINPAR_ENABLE_TIMER_BARRIERS "Add additional MPI_Barrier() instructions for more accurate time measurements." ON)
 
-option(KAMINPAR_ENABLE_THP "Use transparent huge pages for large memory allocations." ON)
+option(KAMINPAR_ENABLE_THP "Use transparent huge pages for large memory allocations (Linux only)." ON)
 
 option(KAMINPAR_BUILD_WITH_ASAN "Enable address sanitizer." OFF)
 option(KAMINPAR_BUILD_WITH_UBSAN "Enable undefined behaviour sanitizer." OFF)
@@ -43,24 +43,11 @@ option(KAMINPAR_BUILD_WITH_CCACHE "Use ccache to build." ON)
 option(KAMINPAR_BUILD_WITH_DEBUG_SYMBOLS "Always build with debug symbols, even in Release mode." ON)
 option(KAMINPAR_BUILD_WITH_MTKAHYPAR "If Mt-KaHyPar can be found, build the Mt-KaHyPar initial partitioner." OFF)
 option(KAMINPAR_BUILD_WITH_GROWT "Build the shared-memory partitioner with Growt." ON)
+option(KAMINPAR_BUILD_WITH_SPARSEHASH "Build with Google Sparsehash." OFF)
 option(KAMINPAR_BUILD_WITH_PG "Build with the -pg option for profiling." OFF)
 
-# Control graph compression options
-###################################
-option(KAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING "Use high-degree encoding for the compressed graph." ON)
-option(KAMINPAR_COMPRESSION_INTERVAL_ENCODING "Use interval encoding for the compressed graph." ON)
-option(KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING "Use run-length encoding for the compressed graph." OFF)
-option(KAMINPAR_COMPRESSION_STREAM_ENCODING "Use stream encoding for the compressed graph." OFF)
-option(KAMINPAR_COMPRESSION_FAST_DECODING "Use fast decoding for the compressed graph." OFF)
-option(KAMINPAR_COMPRESSION_ISOLATED_NODES_SEPARATION "Whether all isolated nodes are the last nodes of the input graph" OFF)
-
-if (KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING AND KAMINPAR_COMPRESSION_STREAM_ENCODING)
-    message(FATAL_ERROR "Either run-length or stream encoding can be used for varints but not both.")
-endif ()
-
 # Control data type sizes
 #########################
-
 # These IDs refer to the shared-memory partitioner + local IDs of the distributed partitioner
 option(KAMINPAR_64BIT_IDS "Use 64 bits for node and edge IDs." OFF)
 option(KAMINPAR_64BIT_EDGE_IDS "Use 64 bits for edge IDs." OFF)
@@ -76,9 +63,34 @@ option(KAMINPAR_64BIT_LOCAL_WEIGHTS "Use 64 bit for local node and edge weights.
 # which is copied to each PE and build with data types of the shared-memory partitioner.
 # Thus, force 64 bit weights for the shared-memory partitioner in this case.
 if (KAMINPAR_BUILD_DISTRIBUTED)
+    message(STATUS "Distributed build: enabling 64 bit weights.")
     set(KAMINPAR_64BIT_WEIGHTS ON)
 endif ()
 
+# Control graph compression options
+###################################
+option(KAMINPAR_COMPRESSION_EDGE_WEIGHTS "Whether to compress edge weights." ON)
+option(KAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING "Use high-degree encoding for the compressed graph." ON)
+option(KAMINPAR_COMPRESSION_INTERVAL_ENCODING "Use interval encoding for the compressed graph." ON)
+option(KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING "Use run-length encoding for the compressed graph." OFF)
+option(KAMINPAR_COMPRESSION_STREAM_ENCODING "Use stream encoding for the compressed graph." OFF)
+option(KAMINPAR_COMPRESSION_FAST_DECODING "Use fast decoding for the compressed graph." OFF)
+option(KAMINPAR_COMPRESSION_ISOLATED_NODES_SEPARATION "Whether all isolated nodes are the last nodes of the input graph" OFF)
+
+if (KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING AND KAMINPAR_COMPRESSION_STREAM_ENCODING)
+    message(FATAL_ERROR "Either run-length or stream encoding can be used for varints but not both.")
+endif ()
+
+if (KAMINPAR_64BIT_NODE_IDS AND KAMINPAR_COMPRESSION_STREAM_ENCODING)
+    message(FATAL_ERROR "Stream encoding cannot be used with 64-bit NodeIDs.")
+endif ()
+
+if (KAMINPAR_COMPRESSION_EDGE_WEIGHTS AND KAMINPAR_COMPRESSION_STREAM_ENCODING)
+    message(FATAL_ERROR "Stream encoding cannot be used together with compressed edge weights.")
+elseif (KAMINPAR_COMPRESSION_EDGE_WEIGHTS AND KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING)
+    message(FATAL_ERROR "Run-length encoding cannot be used together with compressed edge weights.")
+endif ()
+
 ################################################################################
 ## Declare dependencies                                                       ##
 ################################################################################
@@ -125,27 +137,18 @@ endif ()
 
 # Always enable Debug symbols (including in Release mode)
 if (KAMINPAR_BUILD_WITH_DEBUG_SYMBOLS)
-    add_compile_options(-g -g3)
+    add_compile_options(-g)
 endif ()
 
 # Set compile flags
-add_compile_options(-msse4.1)
+check_cxx_compiler_flag(-msse4.1 COMPILER_SUPPORTS_MSSE41)
+if (COMPILER_SUPPORTS_MSSE41)
+    add_compile_options(-msse4.1)
+endif ()
 
 check_cxx_compiler_flag(-mcx16 COMPILER_SUPPORTS_MCX16)
 if (COMPILER_SUPPORTS_MCX16)
     add_compile_options(-mcx16)
-else ()
-    message(WARNING "-mcx16 flag not supported by the compiler")
-
-    if (KAMINPAR_BUILD_WITH_GROWT)
-        message(WARNING "-mcx16 flag not supported by the compiler: cannot use growt for the shared-memory partitioner")
-        set(KAMINPAR_BUILD_WITH_GROWT OFF)
-    endif ()
-
-    if (KAMINPAR_BUILD_DISTRIBUTED)
-        message(WARNING "-mcx16 flag not supported by the compiler: cannot build the distributed partitioner")
-        set(KAMINPAR_BUILD_DISTRIBUTED OFF)
-    endif ()
 endif ()
 
 if (KAMINPAR_BUILD_WITH_MTUNE_NATIVE) 
@@ -214,6 +217,13 @@ endif ()
 
 message(STATUS "Graph compression summary:")
 
+if (KAMINPAR_COMPRESSION_EDGE_WEIGHTS)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_EDGE_WEIGHTS")
+    message("  Compressed edge weights: enabled")
+else ()
+    message("  Compressed edge weights: disabled")
+endif ()
+
 if (KAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING)
     list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING")
     message("  High-degree encoding: enabled")
@@ -273,38 +283,35 @@ endif ()
 
 if (KAMINPAR_64BIT_WEIGHTS)
     list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_64BIT_WEIGHTS")
-    set(KAMINPAR_SHM_NODE_WEIGHT_STR "std::int64_t")
-    set(KAMINPAR_SHM_EDGE_WEIGHT_STR "std::int64_t")
+    set(KAMINPAR_SHM_WEIGHT_STR "std::int64_t")
 else () 
-    set(KAMINPAR_SHM_NODE_WEIGHT_STR "std::int32_t")
-    set(KAMINPAR_SHM_EDGE_WEIGHT_STR "std::int32_t")
+    set(KAMINPAR_SHM_WEIGHT_STR "std::int32_t")
 endif ()
 
 if (KAMINPAR_64BIT_LOCAL_WEIGHTS)
     list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_64BIT_LOCAL_WEIGHTS")
-    set(KAMINPAR_DIST_NODE_WEIGHT_STR "std::int64_t")
-    set(KAMINPAR_DIST_EDGE_WEIGHT_STR "std::int64_t")
+    set(KAMINPAR_DIST_WEIGHT_STR "std::int64_t")
 else ()
-    set(KAMINPAR_DIST_NODE_WEIGHT_STR "std::int32_t")
-    set(KAMINPAR_DIST_EDGE_WEIGHT_STR "std::int32_t")
+    set(KAMINPAR_DIST_WEIGHT_STR "std::int32_t")
 endif ()
 
 message(STATUS "Data type summary:")
-message("  {shm, dist}::NodeID    = ${KAMINPAR_SHM_NODE_ID_STR} | {shm, dist}::EdgeID    = ${KAMINPAR_SHM_EDGE_ID_STR}")
-message("  dist::GlobalNodeID     = std::uint64_t | dist::GlobalEdgeID     = std::uint64_t")
-message("  shm::NodeWeight        =  ${KAMINPAR_SHM_NODE_WEIGHT_STR} | shm::EdgeWeight        =  ${KAMINPAR_SHM_EDGE_WEIGHT_STR}")
-message("  dist::NodeWeight       =  ${KAMINPAR_DIST_NODE_WEIGHT_STR} | dist::EdgeWeight       =  ${KAMINPAR_DIST_EDGE_WEIGHT_STR}")
-message("  dist::GlobalNodeWeight =  std::int64_t | dist::GlobalEdgeWeight =  std::int64_t")
-message("  {shm, dist}::BlockID   = std::uint32_t")
-message("  shm::BlockWeight       =  ${KAMINPAR_SHM_NODE_WEIGHT_STR}")
-message("  dist::BlockWeight      =  std::int64_t")
+message("  {shm, dist}::NodeID: ${KAMINPAR_SHM_NODE_ID_STR}")
+message("  {shm, dist}::EdgeID: ${KAMINPAR_SHM_EDGE_ID_STR}")
+message("  shm::{Node, Edge}Weight: ${KAMINPAR_SHM_WEIGHT_STR}")
+message("  {dist::Global{Node, Edge}ID: std::uint64_t")
+message("  dist::Global{Node, Edge}Weight: std::int64_t")
+message("  dist::{Node, Edge}Weight: ${KAMINPAR_DIST_WEIGHT_STR}")
 
 ################################################################################
 ## Search and fetch dependencies                                              ##
 ################################################################################
 
 # Google Sparsehash 
-find_package(Sparsehash REQUIRED)
+if (KAMINPAR_BUILD_WITH_SPARSEHASH)
+    find_package(Sparsehash REQUIRED)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_SPARSEHASH_FOUND")
+endif ()
 
 if (KAMINPAR_BUILD_WITH_CCACHE)
     find_program(CCACHE_PROGRAM ccache)
@@ -315,7 +322,9 @@ endif ()
 
 if (KAMINPAR_BUILD_WITH_GROWT)
     list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_USES_GROWT")
+endif ()
 
+if (TRUE) 
     add_subdirectory(external/growt EXCLUDE_FROM_ALL)
     add_library(growt INTERFACE)
     target_include_directories(growt SYSTEM INTERFACE "external/growt")

CMakePresets.json

@@ -21,14 +21,15 @@
             "displayName": "Default Config for dKaMinPar",
             "cacheVariables": {
                 "KAMINPAR_BUILD_DISTRIBUTED": "ON",
+                "KAMINPAR_BUILD_WITH_SPARSEHASH": "ON",
                 "KAMINPAR_64BIT_IDS": "OFF",
                 "KAMINPAR_64BIT_EDGE_IDS": "OFF",
                 "KAMINPAR_64BIT_NODE_IDS": "OFF",
                 "KAMINPAR_64BIT_WEIGHTS": "ON"
             }
         },
         {
-            "name": "compressed",
+            "name": "memory",
             "displayName": "Default Config for KaMinPar with Memory Optimizations",
             "cacheVariables": {
                 "KAMINPAR_64BIT_EDGE_IDS": "ON",
@@ -49,8 +50,8 @@
             "inherits": ["default", "stats"]
         },
         {
-            "name": "compressed-stats",
-            "inherits": ["compressed", "stats"]
+            "name": "memory-stats",
+            "inherits": ["memory", "stats"]
         },
         {
             "name": "distributed-stats",

README.MD

@@ -11,17 +11,18 @@ Moreover, for large values of k, it is an order of magnitude faster than competi
 
 ### Requirements
 
-* C++17 compiler (GCC or Clang)
-* CMake 
-* Intel Thread Building Blocks library (TBB)
-* MPI (optional)
+* **Compiler:** GCC or Clang with C++20 support
+* **CPU:** x86 or ARM
+* **Operating System:** Linux or macOS
+* **Tools:** CMake
+* **Libraries:** Intel TBB, MPI (optional, for the distributed partitioner)
 
 ### Building KaMinPar
 
 Build KaMinPar following the standard CMake steps:
 
 ```shell
-cmake -B build -DCMAKE_BUILD_TYPE=Release --preset=<default|distributed>
+cmake -B build -DCMAKE_BUILD_TYPE=Release --preset=<default|memory|distributed>
 cmake --build build --parallel
 ```
 
@@ -31,7 +32,7 @@ To partition a graph in METIS format using (d)KaMinPar, run
 
 ```shell
 # KaMinPar: shared-memory partitioning
-./build/apps/KaMinPar [-P fast|default|strong|largek] -G <graph filename> -k <number of blocks> -t <nproc> [--epsilon=0.03] [--seed=0]
+./build/apps/KaMinPar [-P default|strong|memory|largek] -G <graph filename> -k <number of blocks> -t <nproc> [--epsilon=0.03] [--seed=0]
 
 # dKaMinPar: distributed partitioning
 mpirun -n <nproc> ./build/apps/dKaMinPar [-P default|strong] -G <graph filename> -k <number of blocks> [--epsilon=0.03] [--seed=0]

apps/CMakeLists.txt

@@ -1,9 +1,10 @@
 set(KAMINPAR_IO_SOURCE_FILES
     io/file_tokener.h
-    io/metis_parser.h
-    io/metis_parser.cc
-    io/parhip_parser.h
-    io/parhip_parser.cc
+    io/binary_util.h
+    io/shm_metis_parser.h
+    io/shm_metis_parser.cc
+    io/shm_parhip_parser.h
+    io/shm_parhip_parser.cc
     io/shm_compressed_graph_binary.h
     io/shm_compressed_graph_binary.cc
     io/shm_input_validator.h
@@ -39,7 +40,11 @@ add_shm_app(KaMinPar KaMinPar.cc)
 if (TARGET kaminpar_dist)
     add_dist_app(dKaMinPar dKaMinPar.cc)
     target_sources(dKaMinPar PRIVATE 
-        ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_io.cc)
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_io.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_metis_parser.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_metis_parser.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_parhip_parser.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/dist_parhip_parser.cc)
     target_link_libraries(dKaMinPar PRIVATE KaGen::KaGen)
 endif ()
 

apps/benchmarks/dist_coarsening_benchmark.cc

@@ -9,15 +9,12 @@
 #include <kaminpar-cli/dkaminpar_arguments.h>
 // clang-format on
 
-#include <fstream>
-
 #include <mpi.h>
 #include <omp.h>
 
 #include "kaminpar-dist/coarsening/coarsener.h"
 #include "kaminpar-dist/context.h"
-
-#include "kaminpar-shm/kaminpar.h"
+#include "kaminpar-dist/factories.h"
 
 #include "kaminpar-common/logger.h"
 #include "kaminpar-common/timer.h"
@@ -58,13 +55,15 @@ int main(int argc, char *argv[]) {
   auto &graph = *wrapper.graph;
   ctx.partition.graph = std::make_unique<GraphContext>(graph, ctx.partition);
 
-  Coarsener coarsener(graph, ctx);
+  auto coarsener = factory::create_coarsener(ctx);
   const DistributedGraph *c_graph = &graph;
+  coarsener->initialize(c_graph);
 
   while (c_graph->global_n() > ctx.partition.k * ctx.coarsening.contraction_limit ||
-         (min_levels > 0 && coarsener.level() < min_levels)) {
-    const DistributedGraph *new_c_graph = coarsener.coarsen_once();
-    if (new_c_graph == c_graph) {
+         (min_levels > 0 && coarsener->level() < min_levels)) {
+    const bool converged = coarsener->coarsen();
+    const DistributedGraph *new_c_graph = &coarsener->current();
+    if (converged) {
       LOG << "=> converged";
       break;
     }
@@ -74,7 +73,7 @@ int main(int argc, char *argv[]) {
     LOG << "=> n=" << c_graph->global_n() << " m=" << c_graph->global_m()
         << " max_node_weight=" << c_graph->max_node_weight();
 
-    if (max_levels > 0 && coarsener.level() == max_levels) {
+    if (max_levels > 0 && coarsener->level() == max_levels) {
       LOG << "=> number of configured levels reached";
       break;
     }

apps/benchmarks/dist_contraction_benchmark.cc

@@ -9,21 +9,14 @@
 #include <kaminpar-cli/dkaminpar_arguments.h>
 // clang-format on
 
-#include <fstream>
-
 #include <mpi.h>
 #include <omp.h>
 
-#include "kaminpar-dist/coarsening/contraction/cluster_contraction.h"
+#include "kaminpar-dist/coarsening/contraction/global_cluster_contraction.h"
 #include "kaminpar-dist/context.h"
 #include "kaminpar-dist/dkaminpar.h"
-#include "kaminpar-dist/factories.h"
-#include "kaminpar-dist/graphutils/communication.h"
-#include "kaminpar-dist/metrics.h"
-#include "kaminpar-dist/presets.h"
 
 #include "kaminpar-common/logger.h"
-#include "kaminpar-common/random.h"
 #include "kaminpar-common/timer.h"
 
 #include "apps/benchmarks/dist_io.h"
@@ -55,16 +48,16 @@ int main(int argc, char *argv[]) {
   auto &graph = *wrapper.graph;
   ctx.partition.graph = std::make_unique<GraphContext>(graph, ctx.partition);
 
-  GlobalClustering clustering =
-      load_node_property_vector<NoinitVector<GlobalNodeID>>(graph, clustering_filename);
+  using GlobalClustering = StaticArray<GlobalNodeID>;
+  auto clustering = load_node_property_vector<GlobalClustering>(graph, clustering_filename);
 
   // Compute coarse graph
   START_TIMER("Contraction");
   const auto result = contract_clustering(graph, clustering, ctx.coarsening);
   STOP_TIMER();
 
   LOG << "Coarse graph:";
-  print_graph_summary(result.graph);
+  print_graph_summary(result->get());
 
   // Output statistics
   mpi::barrier(MPI_COMM_WORLD);

apps/benchmarks/dist_io.h

@@ -102,7 +102,7 @@ inline DistributedGraphWrapper load_graph(const std::string &graph_name) {
   auto [global_to_ghost, ghost_to_global, ghost_owner] = mapper.finalize();
 
   DistributedGraphWrapper wrapper;
-  wrapper.graph = std::make_unique<DistributedGraph>(
+  wrapper.graph = std::make_unique<DistributedGraph>(std::make_unique<DistributedCSRGraph>(
       std::move(node_distribution),
       std::move(edge_distribution),
       std::move(nodes),
@@ -114,7 +114,7 @@ inline DistributedGraphWrapper load_graph(const std::string &graph_name) {
       std::move(global_to_ghost),
       false,
       MPI_COMM_WORLD
-  );
+  ));
   return wrapper;
 }