Merge pull request nmslib#4 from chroma-core/hammad/persist

This PR adds a batch persist functionality via the persistDirty() nethid to the graph which only persists dirtied elements in the graph. We store data in four files, a header, the data_level_0, the length, and the link lists. The latter three files map to the in memory representation. Data is never read from disk except on load, serving as a write-through cache. Callers are expected to periodically call persistDirty() in a thread-compatible way.

This storage scheme is extremely naive, and is only meant as an improvement to serializing the whole index. We can make many improvements in terms of disk access, layout, caching, and durability.

.github/workflows/build.yml

@@ -74,4 +74,5 @@ jobs:
           ./getUnormalized_test
           ./test_updates
           ./test_updates update
+          ./persistent_test
         shell: bash

CMakeLists.txt

@@ -9,7 +9,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     set(CMAKE_CXX_STANDARD 11)
 
     if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-      SET( CMAKE_CXX_FLAGS  "-Ofast -DNDEBUG -std=c++11 -DHAVE_CXX0X -openmp -march=native -fpic -ftree-vectorize")
+      SET( CMAKE_CXX_FLAGS "-Ofast -DNDEBUG -std=c++11 -DHAVE_CXX0X -openmp -march=native -fpic -ftree-vectorize")
     elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
       SET( CMAKE_CXX_FLAGS  "-Ofast -lrt -DNDEBUG -std=c++11 -DHAVE_CXX0X -march=native -fpic -w -fopenmp -ftree-vectorize -ftree-vectorizer-verbose=0" )
     elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
@@ -56,5 +56,8 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
 
     add_executable(getUnormalized_test tests/cpp/getUnormalized_test.cpp)
     target_link_libraries(getUnormalized_test hnswlib)
+
+    add_executable(persistent_test tests/cpp/persistent_test.cpp)
+    target_link_libraries(persistent_test hnswlib)
 endif()