Merge branch 'develop'

.clangd

@@ -1,2 +1,2 @@
 CompileFlags:
-  Add: [-std=c++17]
+  Add: [-std=c++20]

.gitignore

@@ -1,5 +1,7 @@
 compile_commands.json
+layout.kdl
 .idea
+.vscode
 *~
 cmake-build-*
 build*/

CMakeLists.txt

@@ -10,7 +10,7 @@ project(KaMinPar
 
 set(PROJECT_VENDOR "Daniel Seemaier")
 set(PROJECT_CONTACT "daniel.seemaier@kit.edu")
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 
 ################################################################################
 ## Options                                                                    ##
@@ -22,12 +22,15 @@ option(KAMINPAR_BUILD_TESTS "Build unit tests" ON)
 option(KAMINPAR_BUILD_DISTRIBUTED "Build distributed partitioner." OFF)
 option(KAMINPAR_BUILD_APPS "Build binaries." ON)
 option(KAMINPAR_BUILD_BENCHMARKS "Build benchmark binaries." OFF)
+option(KAMINPAR_BUILD_TOOLS "Build tool binaries." OFF)
 
 option(KAMINPAR_BUILD_EXPERIMENTAL_FEATURES "Include experimental features in the build. This might increase compile times drastically." OFF)
 
 # Control how to build
 ######################
-option(KAMINPAR_ENABLE_STATISTICS "Generate and output detailed statistics." ON)
+option(KAMINPAR_ENABLE_HEAP_PROFILING "Profile and output heap memory usage." OFF)
+option(KAMINPAR_ENABLE_PAGE_PROFILING "Profile pages allocated via mmap." OFF)
+option(KAMINPAR_ENABLE_STATISTICS "Generate and output detailed statistics." OFF)
 option(KAMINPAR_ENABLE_TIMERS "Measure running times. Must be set to 'OFF' if the library interface is used from multiple threads simulatinously." ON)
 option(KAMINPAR_ENABLE_TIMER_BARRIERS "Add additional MPI_Barrier() instructions for more accurate time measurements." ON)
 
@@ -37,8 +40,22 @@ option(KAMINPAR_BUILD_WITH_MTUNE_NATIVE "Build with -mtune=native." ON)
 option(KAMINPAR_BUILD_WITH_CCACHE "Use ccache to build." ON)
 option(KAMINPAR_BUILD_WITH_DEBUG_SYMBOLS "Always build with debug symbols, even in Release mode." ON)
 option(KAMINPAR_BUILD_WITH_MTKAHYPAR "If Mt-KaHyPar can be found, build the Mt-KaHyPar initial partitioner." OFF)
+option(KAMINPAR_BUILD_WITH_GROWT "Build the shared-memory partitioner with Growt." ON)
 option(KAMINPAR_BUILD_WITH_PG "Build with the -pg option for profiling." OFF)
 
+# Control graph compression options
+###################################
+option(KAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING "Use high-degree encoding for the compressed graph." ON)
+option(KAMINPAR_COMPRESSION_INTERVAL_ENCODING "Use interval encoding for the compressed graph." ON)
+option(KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING "Use run-length encoding for the compressed graph." OFF)
+option(KAMINPAR_COMPRESSION_STREAM_ENCODING "Use stream encoding for the compressed graph." OFF)
+option(KAMINPAR_COMPRESSION_FAST_DECODING "Use fast decoding for the compressed graph." OFF)
+option(KAMINPAR_COMPRESSION_ISOLATED_NODES_SEPARATION "Whether all isolated nodes are the last nodes of the input graph" OFF)
+
+if (KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING AND KAMINPAR_COMPRESSION_STREAM_ENCODING)
+    message(FATAL_ERROR "Either run-length or stream encoding can be used for varints but not both.")
+endif ()
+
 # Control data type sizes
 #########################
 
@@ -109,7 +126,26 @@ if (KAMINPAR_BUILD_WITH_DEBUG_SYMBOLS)
     add_compile_options(-g -g3)
 endif ()
 
-# Set compile flags 
+# Set compile flags
+add_compile_options(-msse4.1)
+
+check_cxx_compiler_flag(-mcx16 COMPILER_SUPPORTS_MCX16)
+if (COMPILER_SUPPORTS_MCX16)
+    add_compile_options(-mcx16)
+else ()
+    message(WARNING "-mcx16 flag not supported by the compiler")
+
+    if (KAMINPAR_BUILD_WITH_GROWT)
+        message(WARNING "-mcx16 flag not supported by the compiler: cannot use growt for the shared-memory partitioner")
+        set(KAMINPAR_BUILD_WITH_GROWT OFF)
+    endif ()
+
+    if (KAMINPAR_BUILD_DISTRIBUTED)
+        message(WARNING "-mcx16 flag not supported by the compiler: cannot build the distributed partitioner")
+        set(KAMINPAR_BUILD_DISTRIBUTED OFF)
+    endif ()
+endif ()
+
 if (KAMINPAR_BUILD_WITH_MTUNE_NATIVE) 
     add_compile_options(-mtune=native -march=native)
 endif ()
@@ -133,7 +169,24 @@ if (KAMINPAR_ENABLE_STATISTICS)
     list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_ENABLE_STATISTICS")
     message(STATUS "Statistics: enabled")
 else ()
-    message(STATIS "Statistics: disabled")
+    message(STATUS "Statistics: disabled")
+endif ()
+
+if (KAMINPAR_ENABLE_HEAP_PROFILING)
+    string(LENGTH "${CMAKE_SOURCE_DIR}/" SOURCE_PATH_SIZE)
+
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_ENABLE_HEAP_PROFILING")
+    list(APPEND KAMINPAR_DEFINITIONS "-DSOURCE_PATH_SIZE=${SOURCE_PATH_SIZE}")
+    message(STATUS "Heap Profiling: enabled")
+else ()
+    message(STATUS "Heap Profiling: disabled")
+endif ()
+
+if (KAMINPAR_ENABLE_PAGE_PROFILING)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_ENABLE_PAGE_PROFILING")
+    message(STATUS "Page Profiling: enabled")
+else ()
+    message(STATUS "Page Profiling: disabled")
 endif ()
 
 if (KAMINPAR_ENABLE_TIMERS)
@@ -150,6 +203,51 @@ else ()
     message(STATUS "Timer barriers: disabled")
 endif ()
 
+message(STATUS "Graph compression summary:")
+
+if (KAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_HIGH_DEGREE_ENCODING")
+    message("  High-degree encoding: enabled")
+else ()
+    message("  High-degree encoding: disabled")
+endif ()
+
+if (KAMINPAR_COMPRESSION_INTERVAL_ENCODING)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_INTERVAL_ENCODING")
+    message("  Interval encoding: enabled")
+else ()
+    message("  Interval encoding: disabled")
+endif ()
+
+if (KAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_RUN_LENGTH_ENCODING")
+    message("  Run-length encoding: enabled")
+else ()
+    message("  Run-length encoding: disabled")
+endif ()
+
+if (KAMINPAR_COMPRESSION_STREAM_ENCODING)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_STREAM_ENCODING")
+    message("  Stream encoding: enabled")
+else ()
+    message("  Stream encoding: disabled")
+endif ()
+
+if (KAMINPAR_COMPRESSION_FAST_DECODING)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_FAST_DECODING")
+    add_compile_options(-mbmi2)
+    message("  Fast decoding: enabled")
+else ()
+    message("  Fast decoding: disabled")
+endif ()
+
+if (KAMINPAR_COMPRESSION_ISOLATED_NODES_SEPARATION)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_COMPRESSION_ISOLATED_NODES_SEPARATION")
+    message("  Isolated nodes separation: enabled")
+else ()
+    message("  Isolated nodes separation: disabled")
+endif ()
+
 if (KAMINPAR_64BIT_NODE_IDS OR KAMINPAR_64BIT_IDS)
     list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_64BIT_NODE_IDS")
     set(KAMINPAR_SHM_NODE_ID_STR "std::uint64_t")
@@ -203,6 +301,14 @@ if (KAMINPAR_BUILD_WITH_CCACHE)
     endif ()
 endif ()
 
+if (KAMINPAR_BUILD_WITH_GROWT)
+    list(APPEND KAMINPAR_DEFINITIONS "-DKAMINPAR_USES_GROWT")
+
+    add_subdirectory(external/growt EXCLUDE_FROM_ALL)
+    add_library(growt INTERFACE)
+    target_include_directories(growt SYSTEM INTERFACE "external/growt")
+endif ()
+
 if (KAMINPAR_BUILD_DISTRIBUTED)
     # MPI
     set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
@@ -212,19 +318,6 @@ if (KAMINPAR_BUILD_DISTRIBUTED)
         set(KAMINPAR_BUILD_DISTRIBUTED OFF)
     endif ()
 
-    # Growt (needs -mcx16, i.e., does not work on ARM)
-    check_cxx_compiler_flag(-mcx16 COMPILER_SUPPORTS_MCX16)
-    if (COMPILER_SUPPORTS_MCX16) 
-        add_compile_options(-mcx16)
-    else ()
-        message(WARNING "-mcx16 flag not supported by the compiler: cannot build the distributed partitioner")
-        set(KAMINPAR_BUILD_DISTRIBUTED OFF)
-    endif()
-
-    add_subdirectory(external/growt EXCLUDE_FROM_ALL)
-    add_library(growt INTERFACE)
-    target_include_directories(growt SYSTEM INTERFACE "external/growt")
-
     # Google Sparsehash 
     find_package(Sparsehash REQUIRED)
 endif ()
@@ -290,7 +383,7 @@ endif ()
 
 # Unit tests
 if (KAMINPAR_BUILD_TESTS)
-    add_subdirectory(external/googletest EXCLUDE_FROM_ALL)
+    add_subdirectory(external/googletest EXCLUDE_FROM_ALL SYSTEM)
 
     enable_testing()
     add_subdirectory(tests)

CMakePresets.json

@@ -0,0 +1,60 @@
+{
+    "version": 6,
+    "cmakeMinimumRequired": {
+        "major": 3,
+        "minor": 21,
+        "patch": 0
+    },
+    "configurePresets": [
+        {
+            "name": "default",
+            "displayName": "Default Config",
+            "cacheVariables": {
+                "KAMINPAR_64BIT_IDS": "OFF",
+                "KAMINPAR_64BIT_EDGE_IDS": "OFF",
+                "KAMINPAR_64BIT_NODE_IDS": "OFF",
+                "KAMINPAR_64BIT_WEIGHTS": "OFF"
+            }
+        },
+        {
+            "name": "distributed",
+            "displayName": "Default Config for dKaMinPar",
+            "cacheVariables": {
+                "KAMINPAR_BUILD_DISTRIBUTED": "ON",
+                "KAMINPAR_64BIT_IDS": "OFF",
+                "KAMINPAR_64BIT_EDGE_IDS": "OFF",
+                "KAMINPAR_64BIT_NODE_IDS": "OFF",
+                "KAMINPAR_64BIT_WEIGHTS": "ON"
+            }
+        },
+        {
+            "name": "compressed",
+            "displayName": "Default Config for KaMinPar with Memory Optimizations",
+            "cacheVariables": {
+                "KAMINPAR_64BIT_EDGE_IDS": "ON",
+                "KAMINPAR_64BIT_WEIGHTS": "ON"
+            }
+        },
+        {
+            "name": "stats",
+            "displayName": "Default Config for KaMinPar with Statistics",
+            "cacheVariables": {
+                "KAMINPAR_ENABLE_STATISTICS": "ON",
+                "KAMINPAR_ENABLE_HEAP_PROFILING": "ON"
+            }
+        },
+
+        {
+            "name": "default-stats",
+            "inherits": ["default", "stats"]
+        },
+        {
+            "name": "compressed-stats",
+            "inherits": ["compressed", "stats"]
+        },
+        {
+            "name": "distributed-stats",
+            "inherits": ["distributed", "stats"]
+        }
+    ]
+}

README.MD

@@ -21,7 +21,7 @@ Moreover, for large values of k, it is an order of magnitude faster than competi
 Build KaMinPar following the standard CMake steps:
 
 ```shell
-cmake -B build -DCMAKE_BUILD_TYPE=Release -DKAMINPAR_BUILD_DISTRIBUTED=On
+cmake -B build -DCMAKE_BUILD_TYPE=Release --preset=<default|distributed>
 cmake --build build --parallel
 ```
 
@@ -43,7 +43,7 @@ Presets can be viewed by using the `--dump-config` flag; to use a custom preset,
 
 ```shell
 # Write the default preset to a file
-./KaMinPar [-P default|strong|largek] --dump-config > my_preset.ini
+./KaMinPar -P default --dump-config > my_preset.ini
 
 # ... modify the configuration by editing my_preset.ini ...
 

apps/CMakeLists.txt

@@ -1,7 +1,23 @@
+set(KAMINPAR_IO_SOURCE_FILES
+    io/parhip_parser.h
+    io/parhip_parser.cc
+    io/shm_compressed_graph_binary.h
+    io/shm_compressed_graph_binary.cc
+    io/shm_input_validator.h
+    io/shm_input_validator.cc
+    io/shm_io.h
+    io/shm_io.cc)
+
+add_library(kaminpar_io ${KAMINPAR_IO_SOURCE_FILES})
+target_include_directories(kaminpar_io PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/io/../")
+target_link_libraries(kaminpar_io PUBLIC KaMinPar::KaMinPar KaMinPar::KaMinParCLI11)
+
+add_library(KaMinPar::KaMinParIO ALIAS kaminpar_io)
+
 function(add_shm_app target)
     add_executable(${target} ${ARGN})
     target_include_directories(${target} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_link_libraries(${target} PRIVATE KaMinPar::KaMinPar KaMinPar::KaMinParCLI11)
+    target_link_libraries(${target} PRIVATE KaMinPar::KaMinPar KaMinPar::KaMinParCLI11 KaMinPar::KaMinParIO)
     install(TARGETS ${target})
     message(STATUS "Enabled app: ${target}")
 endfunction()
@@ -16,9 +32,6 @@ function(add_dist_app target)
 endfunction()
 
 add_shm_app(KaMinPar KaMinPar.cc)
-target_sources(KaMinPar PRIVATE 
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_io.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/io/shm_input_validator.cc)
 
 if (TARGET kaminpar_dist)
     add_dist_app(dKaMinPar dKaMinPar.cc)
@@ -31,3 +44,6 @@ if (KAMINPAR_BUILD_BENCHMARKS)
     add_subdirectory(benchmarks)
 endif ()
 
+if (KAMINPAR_BUILD_TOOLS)
+    add_subdirectory(tools)
+endif ()