Skip to content

Commit

Permalink
reduce the number of I/O operations for HNSW during deserialization (#…
Browse files Browse the repository at this point in the history
…156)

Signed-off-by: jinjiabao.jjb <jinjiabao.jjb@antgroup.com>
  • Loading branch information
inabao authored and jinjiabao.jjb committed Nov 28, 2024
1 parent e900f25 commit fd8a1e6
Show file tree
Hide file tree
Showing 12 changed files with 271 additions and 369 deletions.
31 changes: 5 additions & 26 deletions src/algorithm/hnswlib/hnswalg.h
Original file line number Diff line number Diff line change
Expand Up @@ -1069,38 +1069,17 @@ class HierarchicalNSW : public AlgorithmInterface<float> {
}
}

// load using reader
void
loadIndex(std::function<void(uint64_t, uint64_t, void*)> read_func,
SpaceInterface* s,
size_t max_elements_i) override {
int64_t cursor = 0;
ReadFuncStreamReader reader(read_func, cursor);
DeserializeImpl(reader, s, max_elements_i);
}

// load index from a file stream
void
loadIndex(std::istream& in_stream, SpaceInterface* s, size_t max_elements_i) override {
IOStreamReader reader(in_stream);
this->DeserializeImpl(reader, s, max_elements_i);
}

// origin load function
void
loadIndex(const std::string& location, SpaceInterface* s, size_t max_elements_i = 0) {
std::ifstream input(location, std::ios::binary);
IOStreamReader reader(input);
this->DeserializeImpl(reader, s, max_elements_i);
input.close();
}

template <typename T>
static void
ReadOne(StreamReader& reader, T& value) {
reader.Read(reinterpret_cast<char*>(&value), sizeof(value));
}

void
loadIndex(StreamReader& buffer_reader, SpaceInterface* s, size_t max_elements_i = 0) override {
this->DeserializeImpl(buffer_reader, s, max_elements_i);
}

void
DeserializeImpl(StreamReader& reader, SpaceInterface* s, size_t max_elements_i = 0) {
ReadOne(reader, offsetLevel0_);
Expand Down
Loading

0 comments on commit fd8a1e6

Please sign in to comment.