Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
alias nlohmann::json to JsonType
Browse files Browse the repository at this point in the history
Signed-off-by: LHT129 <tianlan.lht@antgroup.com>
LHT129 committed Nov 13, 2024
1 parent 83b6c20 commit 58c4ba7
Showing 26 changed files with 69 additions and 64 deletions.
2 changes: 1 addition & 1 deletion src/factory/factory.cpp
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@ Factory::CreateIndex(const std::string& origin_name,
params.use_bsa,
params.use_async_io);
} else if (name == INDEX_HGRAPH) {
auto param = nlohmann::json::parse(parameters);
auto param = JsonType::parse(parameters);
auto common_param = IndexCommonParam::CheckAndCreate(parameters);
if (allocator != nullptr) {
common_param.allocator_ = allocator;
9 changes: 5 additions & 4 deletions src/factory/factory_test.cpp
Original file line number Diff line number Diff line change
@@ -21,13 +21,14 @@
#include <nlohmann/json.hpp>

#include "../logger.h"
#include "typing.h"
#include "vsag/errors.h"

TEST_CASE("create index with full parameters", "[factory][ut]") {
vsag::logger::set_level(vsag::logger::level::debug);

SECTION("hnsw") {
auto parameters = nlohmann::json::parse(R"(
auto parameters = vsag::JsonType::parse(R"(
{
"dtype": "float32",
"metric_type": "l2",
@@ -44,7 +45,7 @@ TEST_CASE("create index with full parameters", "[factory][ut]") {
}

SECTION("diskann") {
auto parameters = nlohmann::json::parse(R"(
auto parameters = vsag::JsonType::parse(R"(
{
"dtype": "float32",
"metric_type": "l2",
@@ -66,7 +67,7 @@ TEST_CASE("create index with full parameters", "[factory][ut]") {
TEST_CASE("create hnsw with incomplete parameters", "[factory][ut]") {
vsag::logger::set_level(vsag::logger::level::debug);

auto standard_parameters = nlohmann::json::parse(R"(
auto standard_parameters = vsag::JsonType::parse(R"(
{
"dtype": "float32",
"metric_type": "l2",
@@ -124,7 +125,7 @@ TEST_CASE("create hnsw with incomplete parameters", "[factory][ut]") {
TEST_CASE("create diskann with incomplete parameters", "[factory][ut]") {
vsag::logger::set_level(vsag::logger::level::debug);

auto standard_parameters = nlohmann::json::parse(R"(
auto standard_parameters = vsag::JsonType::parse(R"(
{
"dim": 256,
"dtype": "float32",
2 changes: 1 addition & 1 deletion src/footer.cpp
Original file line number Diff line number Diff line change
@@ -95,7 +95,7 @@ SerializationFooter::Deserialize(std::istream& in_stream) {

// parse json
std::string serialized_data(buffer.begin(), buffer.begin() + FOOTER_SIZE - sizeof(uint32_t));
json_ = nlohmann::json::parse(serialized_data, nullptr, false);
json_ = JsonType::parse(serialized_data, nullptr, false);
if (json_.is_discarded()) {
throw std::runtime_error("Failed to parse JSON data");
}
3 changes: 2 additions & 1 deletion src/footer.h
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
#pragma once
#include "fmt/format.h"
#include "nlohmann/json.hpp"
#include "typing.h"
#include "vsag/constants.h"

namespace vsag {
@@ -44,7 +45,7 @@ class SerializationFooter {
Deserialize(std::istream& in_stream);

private:
nlohmann::json json_;
JsonType json_;
};

} // namespace vsag
1 change: 1 addition & 0 deletions src/impl/conjugate_graph.h
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@

#include "../footer.h"
#include "../logger.h"
#include "typing.h"
#include "vsag/index.h"

namespace vsag {
10 changes: 5 additions & 5 deletions src/impl/conjugate_graph_test.cpp
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@
#include <fstream>
#include <nlohmann/json.hpp>

#include "../../tests/fixtures/fixtures.h"
#include "fixtures.h"

TEST_CASE("build, add and memory usage", "[ut][conjugate_graph]") {
std::shared_ptr<vsag::ConjugateGraph> conjugate_graph =
@@ -97,7 +97,7 @@ TEST_CASE("serialize and deserialize with binary", "[ut][conjugate_graph]") {
SECTION("deserialize with invalid magic_num") {
vsag::Binary binary = *conjugate_graph->Serialize();

nlohmann::json json;
vsag::JsonType json;
json[vsag::SERIALIZE_MAGIC_NUM] = std::to_string(0xABCD1234);
json[vsag::SERIALIZE_VERSION] = vsag::VERSION;
std::string json_str = json.dump();
@@ -121,7 +121,7 @@ TEST_CASE("serialize and deserialize with binary", "[ut][conjugate_graph]") {
SECTION("deserialize with invalid version") {
vsag::Binary binary = *conjugate_graph->Serialize();

nlohmann::json json;
vsag::JsonType json;
json[vsag::SERIALIZE_MAGIC_NUM] = vsag::MAGIC_NUM;
json[vsag::SERIALIZE_VERSION] = std::to_string(2);
std::string json_str = json.dump();
@@ -178,7 +178,7 @@ TEST_CASE("serialize and deserialize with stream", "[ut][conjugate_graph]") {
REQUIRE(serialize_result.has_value());
out_stream.seekg(conjugate_graph->GetMemoryUsage() - vsag::FOOTER_SIZE, std::ios::beg);

nlohmann::json json;
vsag::JsonType json;
json[vsag::SERIALIZE_MAGIC_NUM] = std::to_string(0xABCD1234);
json[vsag::SERIALIZE_VERSION] = vsag::VERSION;
std::string json_str = json.dump();
@@ -201,7 +201,7 @@ TEST_CASE("serialize and deserialize with stream", "[ut][conjugate_graph]") {
REQUIRE(serialize_result.has_value());
out_stream.seekg(conjugate_graph->GetMemoryUsage() - vsag::FOOTER_SIZE, std::ios::beg);

nlohmann::json json;
vsag::JsonType json;
json[vsag::SERIALIZE_MAGIC_NUM] = vsag::MAGIC_NUM;
json[vsag::SERIALIZE_VERSION] = std::to_string(2);
std::string json_str = json.dump();
2 changes: 1 addition & 1 deletion src/index/diskann.cpp
Original file line number Diff line number Diff line change
@@ -742,7 +742,7 @@ DiskANN::deserialize(const ReaderSet& reader_set) {

std::string
DiskANN::GetStats() const {
nlohmann::json j;
JsonType j;
j[STATSTIC_DATA_NUM] = GetNumElements();
j[STATSTIC_INDEX_NAME] = INDEX_DISKANN;
j[STATSTIC_MEMORY] = GetMemoryUsage();
1 change: 1 addition & 0 deletions src/index/diskann.h
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@
#include "../common.h"
#include "../logger.h"
#include "../utils.h"
#include "typing.h"
#include "vsag/index.h"
#include "vsag/options.h"

22 changes: 11 additions & 11 deletions src/index/diskann_test.cpp
Original file line number Diff line number Diff line change
@@ -159,7 +159,7 @@ TEST_CASE("knn_search", "[diskann][ut]") {
auto query = vsag::Dataset::Make();
query->Dim(dim)->NumElements(1)->Ids(ids.data())->Float32Vectors(vectors.data())->Owner(false);
int64_t k = 10;
nlohmann::json params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}, {"io_limit", 200}}}};
vsag::JsonType params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}, {"io_limit", 200}}}};

SECTION("index empty") {
auto empty_index = std::make_shared<vsag::DiskANN>(diskann::Metric::L2,
@@ -198,30 +198,30 @@ TEST_CASE("knn_search", "[diskann][ut]") {
}

SECTION("invalid parameters diskann not found") {
nlohmann::json invalid_params{};
vsag::JsonType invalid_params{};
auto result = index->KnnSearch(query, k, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
REQUIRE(result.error().type == vsag::ErrorType::INVALID_ARGUMENT);
}

SECTION("invalid parameters beam_search not found") {
nlohmann::json invalid_params{{"diskann", {{"ef_search", 100}, {"io_limit", 200}}}};
vsag::JsonType invalid_params{{"diskann", {{"ef_search", 100}, {"io_limit", 200}}}};

auto result = index->KnnSearch(query, k, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
REQUIRE(result.error().type == vsag::ErrorType::INVALID_ARGUMENT);
}

SECTION("invalid parameters io_limit not found") {
nlohmann::json invalid_params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}}}};
vsag::JsonType invalid_params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}}}};

auto result = index->KnnSearch(query, k, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
REQUIRE(result.error().type == vsag::ErrorType::INVALID_ARGUMENT);
}

SECTION("invalid parameters ef_search not found") {
nlohmann::json invalid_params{{"diskann", {{"beam_search", 4}, {"io_limit", 200}}}};
vsag::JsonType invalid_params{{"diskann", {{"beam_search", 4}, {"io_limit", 200}}}};

auto result = index->KnnSearch(query, k, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
@@ -262,7 +262,7 @@ TEST_CASE("range_search", "[diskann][ut]") {
auto query = vsag::Dataset::Make();
query->Dim(dim)->NumElements(1)->Ids(ids.data())->Float32Vectors(vectors.data())->Owner(false);
float radius = 9.9f;
nlohmann::json params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}, {"io_limit", 200}}}};
vsag::JsonType params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}, {"io_limit", 200}}}};

SECTION("successful case with smaller range_search_limit") {
int64_t range_search_limit = num_elements - 1;
@@ -340,30 +340,30 @@ TEST_CASE("range_search", "[diskann][ut]") {
}

SECTION("invalid parameters diskann not found") {
nlohmann::json invalid_params{};
vsag::JsonType invalid_params{};
auto result = index->RangeSearch(query, radius, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
REQUIRE(result.error().type == vsag::ErrorType::INVALID_ARGUMENT);
}

SECTION("invalid parameters beam_search not found") {
nlohmann::json invalid_params{{"diskann", {{"ef_search", 100}, {"io_limit", 200}}}};
vsag::JsonType invalid_params{{"diskann", {{"ef_search", 100}, {"io_limit", 200}}}};

auto result = index->RangeSearch(query, radius, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
REQUIRE(result.error().type == vsag::ErrorType::INVALID_ARGUMENT);
}

SECTION("invalid parameters io_limit not found") {
nlohmann::json invalid_params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}}}};
vsag::JsonType invalid_params{{"diskann", {{"ef_search", 100}, {"beam_search", 4}}}};

auto result = index->RangeSearch(query, radius, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
REQUIRE(result.error().type == vsag::ErrorType::INVALID_ARGUMENT);
}

SECTION("invalid parameters ef_search not found") {
nlohmann::json invalid_params{{"diskann", {{"beam_search", 4}, {"io_limit", 200}}}};
vsag::JsonType invalid_params{{"diskann", {{"beam_search", 4}, {"io_limit", 200}}}};

auto result = index->RangeSearch(query, radius, invalid_params.dump());
REQUIRE_FALSE(result.has_value());
@@ -469,7 +469,7 @@ TEST_CASE("split building process", "[diskann][ut]") {
}
}

nlohmann::json parameters{
vsag::JsonType parameters{
{"diskann", {{"ef_search", 10}, {"beam_search", 4}, {"io_limit", 20}}}};
float correct = 0;
for (int i = 0; i < num_elements; i++) {
4 changes: 2 additions & 2 deletions src/index/diskann_zparameters.cpp
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ namespace vsag {

CreateDiskannParameters
CreateDiskannParameters::FromJson(const std::string& json_string) {
nlohmann::json params = nlohmann::json::parse(json_string);
JsonType params = JsonType::parse(json_string);
CreateDiskannParameters obj;

auto index_common_param = IndexCommonParam::CheckAndCreate(json_string);
@@ -113,7 +113,7 @@ CreateDiskannParameters::FromJson(const std::string& json_string) {

DiskannSearchParameters
DiskannSearchParameters::FromJson(const std::string& json_string) {
nlohmann::json params = nlohmann::json::parse(json_string);
JsonType params = JsonType::parse(json_string);

DiskannSearchParameters obj;

2 changes: 1 addition & 1 deletion src/index/hgraph_index.cpp
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ empty_binaryset() {
return bs;
}

HGraphIndex::HGraphIndex(const nlohmann::json& index_param,
HGraphIndex::HGraphIndex(const JsonType& index_param,
const vsag::IndexCommonParam& common_param) noexcept
: index_param_(index_param),
common_param_(common_param),
4 changes: 2 additions & 2 deletions src/index/hgraph_index.h
Original file line number Diff line number Diff line change
@@ -45,7 +45,7 @@ class HGraphIndex : public Index {
Vector<std::pair<float, InnerIdType>>,
CompareByFirst>;

HGraphIndex(const nlohmann::json& index_param, const IndexCommonParam& common_param) noexcept;
HGraphIndex(const JsonType& index_param, const IndexCommonParam& common_param) noexcept;

void
Init();
@@ -177,7 +177,7 @@ class HGraphIndex : public Index {
int64_t dim_{0};
MetricType metric_{MetricType::METRIC_TYPE_L2SQR};

const nlohmann::json index_param_{};
const JsonType index_param_{};
const IndexCommonParam common_param_{};

private:
4 changes: 2 additions & 2 deletions src/index/hgraph_index_test.cpp
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@ TEST_CASE("build with allocator", "[ut][hgraphindex]") {
}
}
})";
auto json_param = nlohmann::json::parse(json_str);
auto json_param = vsag::JsonType::parse(json_str);
vsag::IndexCommonParam param;
auto allocator = std::make_shared<vsag::DefaultAllocator>();
param.dim_ = 128;
@@ -92,7 +92,7 @@ TEST_CASE("build with allocator", "[ut][hgraphindex]") {
->Dim(param.dim_)
->Float32Vectors(vectors.data() + new_id * param.dim_)
->Owner(false);
nlohmann::json params{
vsag::JsonType params{
{"hnsw", {{"ef_search", 100}}},
};
auto result2 = index->KnnSearch(query, 1, params.dump());
4 changes: 2 additions & 2 deletions src/index/hgraph_zparameters.cpp
Original file line number Diff line number Diff line change
@@ -38,15 +38,15 @@ HGraphParameters::ParseStringParam(const std::string& str) {
if (str == "") {
return;
}
auto json_obj = nlohmann::json::parse(str);
auto json_obj = JsonType::parse(str);
for (const auto& [key, value] : json_obj.items()) {
this->CheckAndSetKeyValue(key, value);
}
this->refresh_string_by_json();
}

void
HGraphParameters::CheckAndSetKeyValue(const std::string& key, nlohmann::json& value) {
HGraphParameters::CheckAndSetKeyValue(const std::string& key, JsonType& value) {
const auto& iter = EXTERNAL_MAPPING.find(key);
if (iter != EXTERNAL_MAPPING.end()) {
const auto& vec = iter->second;
8 changes: 4 additions & 4 deletions src/index/hgraph_zparameters.h
Original file line number Diff line number Diff line change
@@ -29,15 +29,15 @@ class HGraphParameters {
ParseStringParam(const std::string& str);

void
CheckAndSetKeyValue(const std::string& key, nlohmann::json& value);
CheckAndSetKeyValue(const std::string& key, JsonType& value);

std::string
GetString() {
this->refresh_string_by_json();
return this->str_;
}

nlohmann::json
JsonType
GetJson() {
this->refresh_json_by_string();
return this->json_;
@@ -46,7 +46,7 @@ class HGraphParameters {
private:
inline void
refresh_json_by_string() {
this->json_ = nlohmann::json::parse(str_);
this->json_ = JsonType::parse(str_);
}

inline void
@@ -55,7 +55,7 @@ class HGraphParameters {
}

private:
nlohmann::json json_;
JsonType json_;

std::string str_{DEFAULT_HGRAPH_PARAMS};

2 changes: 1 addition & 1 deletion src/index/hnsw.cpp
Original file line number Diff line number Diff line change
@@ -587,7 +587,7 @@ HNSW::deserialize(std::istream& in_stream) {

std::string
HNSW::GetStats() const {
nlohmann::json j;
JsonType j;
j[STATSTIC_DATA_NUM] = GetNumElements();
j[STATSTIC_INDEX_NAME] = INDEX_HNSW;
j[STATSTIC_MEMORY] = GetMemoryUsage();
Loading

0 comments on commit 58c4ba7

Please sign in to comment.