diff --git a/src/search/executors/filter_executor.h b/src/search/executors/filter_executor.h index 6820ae586eb..ea860fc642d 100644 --- a/src/search/executors/filter_executor.h +++ b/src/search/executors/filter_executor.h @@ -74,7 +74,7 @@ struct QueryExprEvaluator { StatusOr Visit(TagContainExpr *v) const { auto val = GET_OR_RET(ctx->Retrieve(row, v->field->info)); - auto meta = v->field->info->MetadataAs(); + auto meta = v->field->info->MetadataAs(); auto split = util::Split(val, std::string(1, meta->separator)); return std::find(split.begin(), split.end(), v->tag->val) != split.end(); diff --git a/src/search/executors/numeric_field_scan_executor.h b/src/search/executors/numeric_field_scan_executor.h index 970ef10666f..e9699aa4710 100644 --- a/src/search/executors/numeric_field_scan_executor.h +++ b/src/search/executors/numeric_field_scan_executor.h @@ -38,35 +38,38 @@ struct NumericFieldScanExecutor : ExecutorNode { util::UniqueIterator iter{nullptr}; IndexInfo *index; - std::string ns_key; + redis::SearchKey search_key; NumericFieldScanExecutor(ExecutorContext *ctx, NumericFieldScan *scan) - : ExecutorNode(ctx), scan(scan), ss(ctx->storage), index(scan->field->info->index) { - ns_key = ComposeNamespaceKey(index->ns, index->name, ctx->storage->IsSlotIdEncoded()); - } + : ExecutorNode(ctx), + scan(scan), + ss(ctx->storage), + index(scan->field->info->index), + search_key(index->ns, index->name, scan->field->name) {} - std::string IndexKey(double num) { - return InternalKey(ns_key, redis::ConstructNumericFieldSubkey(scan->field->name, num, {}), index->metadata.version, - ctx->storage->IsSlotIdEncoded()) - .Encode(); - } + std::string IndexKey(double num) const { return search_key.ConstructNumericFieldData(num, {}); } - bool InRangeDecode(Slice key, Slice field, double num, double *curr, Slice *user_key) { - auto ikey = InternalKey(key, ctx->storage->IsSlotIdEncoded()); - if (ikey.GetVersion() != index->metadata.version) return false; - auto subkey = ikey.GetSubKey(); + bool InRangeDecode(Slice key, double *curr, Slice *user_key) const { + uint8_t ns_size = 0; + if (!GetFixed8(&key, &ns_size)) return false; + if (ns_size != index->ns.size()) return false; + if (!key.starts_with(index->ns)) return false; + key.remove_prefix(ns_size); - uint8_t flag = 0; - if (!GetFixed8(&subkey, &flag)) return false; - if (flag != (uint8_t)redis::SearchSubkeyType::NUMERIC_FIELD) return false; + uint8_t subkey_type = 0; + if (!GetFixed8(&key, &subkey_type)) return false; + if (subkey_type != (uint8_t)redis::SearchSubkeyType::FIELD) return false; Slice value; - if (!GetSizedString(&subkey, &value)) return false; - if (value != field) return false; + if (!GetSizedString(&key, &value)) return false; + if (value != index->name) return false; + + if (!GetSizedString(&key, &value)) return false; + if (value != scan->field->name) return false; - if (!GetDouble(&subkey, curr)) return false; + if (!GetDouble(&key, curr)) return false; - if (!GetSizedString(&subkey, user_key)) return false; + if (!GetSizedString(&key, user_key)) return false; return true; } @@ -90,7 +93,7 @@ struct NumericFieldScanExecutor : ExecutorNode { double curr = 0; Slice user_key; - if (!InRangeDecode(iter->key(), scan->field->name, scan->range.r, &curr, &user_key)) { + if (!InRangeDecode(iter->key(), &curr, &user_key)) { return end; } diff --git a/src/search/executors/tag_field_scan_executor.h b/src/search/executors/tag_field_scan_executor.h index ddedffc9379..9eff2e66aa5 100644 --- a/src/search/executors/tag_field_scan_executor.h +++ b/src/search/executors/tag_field_scan_executor.h @@ -38,35 +38,37 @@ struct TagFieldScanExecutor : ExecutorNode { util::UniqueIterator iter{nullptr}; IndexInfo *index; - std::string ns_key; std::string index_key; TagFieldScanExecutor(ExecutorContext *ctx, TagFieldScan *scan) - : ExecutorNode(ctx), scan(scan), ss(ctx->storage), index(scan->field->info->index) { - ns_key = ComposeNamespaceKey(index->ns, index->name, ctx->storage->IsSlotIdEncoded()); - index_key = InternalKey(ns_key, redis::ConstructTagFieldSubkey(scan->field->name, scan->tag, {}), - index->metadata.version, ctx->storage->IsSlotIdEncoded()) - .Encode(); - } - - bool InRangeDecode(Slice key, Slice field, Slice *user_key) { - auto ikey = InternalKey(key, ctx->storage->IsSlotIdEncoded()); - if (ikey.GetVersion() != index->metadata.version) return false; - auto subkey = ikey.GetSubKey(); - - uint8_t flag = 0; - if (!GetFixed8(&subkey, &flag)) return false; - if (flag != (uint8_t)redis::SearchSubkeyType::TAG_FIELD) return false; + : ExecutorNode(ctx), + scan(scan), + ss(ctx->storage), + index(scan->field->info->index), + index_key(redis::SearchKey(index->ns, index->name, scan->field->name).ConstructTagFieldData(scan->tag, {})) {} + + bool InRangeDecode(Slice key, Slice *user_key) const { + uint8_t ns_size = 0; + if (!GetFixed8(&key, &ns_size)) return false; + if (ns_size != index->ns.size()) return false; + if (!key.starts_with(index->ns)) return false; + key.remove_prefix(ns_size); + + uint8_t subkey_type = 0; + if (!GetFixed8(&key, &subkey_type)) return false; + if (subkey_type != (uint8_t)redis::SearchSubkeyType::FIELD) return false; Slice value; - if (!GetSizedString(&subkey, &value)) return false; - if (value != field) return false; + if (!GetSizedString(&key, &value)) return false; + if (value != index->name) return false; + + if (!GetSizedString(&key, &value)) return false; + if (value != scan->field->name) return false; - Slice tag; - if (!GetSizedString(&subkey, &tag)) return false; - if (tag != scan->tag) return false; + if (!GetSizedString(&key, &value)) return false; + if (value != scan->tag) return false; - if (!GetSizedString(&subkey, user_key)) return false; + if (!GetSizedString(&key, user_key)) return false; return true; } @@ -85,7 +87,7 @@ struct TagFieldScanExecutor : ExecutorNode { } Slice user_key; - if (!InRangeDecode(iter->key(), scan->field->name, &user_key)) { + if (!InRangeDecode(iter->key(), &user_key)) { return end; } diff --git a/src/search/index_info.h b/src/search/index_info.h index 1751549d690..59abc694b31 100644 --- a/src/search/index_info.h +++ b/src/search/index_info.h @@ -33,12 +33,12 @@ struct IndexInfo; struct FieldInfo { std::string name; IndexInfo *index = nullptr; - std::unique_ptr metadata; + std::unique_ptr metadata; - FieldInfo(std::string name, std::unique_ptr &&metadata) + FieldInfo(std::string name, std::unique_ptr &&metadata) : name(std::move(name)), metadata(std::move(metadata)) {} - bool IsSortable() const { return dynamic_cast(metadata.get()) != nullptr; } + bool IsSortable() const { return metadata->IsSortable(); } bool HasIndex() const { return !metadata->noindex; } template @@ -51,12 +51,12 @@ struct IndexInfo { using FieldMap = std::map; std::string name; - SearchMetadata metadata; + redis::IndexMetadata metadata; FieldMap fields; - redis::SearchPrefixesMetadata prefixes; + redis::IndexPrefixes prefixes; std::string ns; - IndexInfo(std::string name, SearchMetadata metadata) : name(std::move(name)), metadata(std::move(metadata)) {} + IndexInfo(std::string name, redis::IndexMetadata metadata) : name(std::move(name)), metadata(std::move(metadata)) {} void Add(FieldInfo &&field) { const auto &name = field.name; diff --git a/src/search/indexer.cc b/src/search/indexer.cc index 4bce72de10c..752d42bd18a 100644 --- a/src/search/indexer.cc +++ b/src/search/indexer.cc @@ -32,16 +32,16 @@ namespace redis { -StatusOr FieldValueRetriever::Create(SearchOnDataType type, std::string_view key, +StatusOr FieldValueRetriever::Create(IndexOnDataType type, std::string_view key, engine::Storage *storage, const std::string &ns) { - if (type == SearchOnDataType::HASH) { + if (type == IndexOnDataType::HASH) { Hash db(storage, ns); std::string ns_key = db.AppendNamespacePrefix(key); HashMetadata metadata(false); auto s = db.GetMetadata(Database::GetOptions{}, ns_key, &metadata); if (!s.ok()) return {Status::NotOK, s.ToString()}; return FieldValueRetriever(db, metadata, key); - } else if (type == SearchOnDataType::JSON) { + } else if (type == IndexOnDataType::JSON) { Json db(storage, ns); std::string ns_key = db.AppendNamespacePrefix(key); JsonMetadata metadata(false); @@ -50,7 +50,7 @@ StatusOr FieldValueRetriever::Create(SearchOnDataType type, if (!s.ok()) return {Status::NotOK, s.ToString()}; return FieldValueRetriever(value); } else { - assert(false && "unreachable code: unexpected SearchOnDataType"); + assert(false && "unreachable code: unexpected IndexOnDataType"); __builtin_unreachable(); } } @@ -111,94 +111,103 @@ StatusOr IndexUpdater::Record(std::string_view key, c return values; } -Status IndexUpdater::UpdateIndex(const std::string &field, std::string_view key, std::string_view original, - std::string_view current, const std::string &ns) const { - if (original == current) { - // the value of this field is unchanged, no need to update - return Status::OK(); +Status IndexUpdater::UpdateTagIndex(std::string_view key, std::string_view original, std::string_view current, + const SearchKey &search_key, const TagFieldMetadata *tag) const { + const char delim[] = {tag->separator, '\0'}; + auto original_tags = util::Split(original, delim); + auto current_tags = util::Split(current, delim); + + auto to_tag_set = [](const std::vector &tags, bool case_sensitive) -> std::set { + if (case_sensitive) { + return {tags.begin(), tags.end()}; + } else { + std::set res; + std::transform(tags.begin(), tags.end(), std::inserter(res, res.begin()), util::ToLower); + return res; + } + }; + + std::set tags_to_delete = to_tag_set(original_tags, tag->case_sensitive); + std::set tags_to_add = to_tag_set(current_tags, tag->case_sensitive); + + for (auto it = tags_to_delete.begin(); it != tags_to_delete.end();) { + if (auto jt = tags_to_add.find(*it); jt != tags_to_add.end()) { + it = tags_to_delete.erase(it); + tags_to_add.erase(jt); + } else { + ++it; + } } - auto iter = info->fields.find(field); - if (iter == info->fields.end()) { - return {Status::NotOK, "No such field to do index updating"}; + if (tags_to_add.empty() && tags_to_delete.empty()) { + // no change, skip index updating + return Status::OK(); } - auto *metadata = iter->second.metadata.get(); auto *storage = indexer->storage; - auto ns_key = ComposeNamespaceKey(ns, info->name, storage->IsSlotIdEncoded()); - if (auto tag = dynamic_cast(metadata)) { - const char delim[] = {tag->separator, '\0'}; - auto original_tags = util::Split(original, delim); - auto current_tags = util::Split(current, delim); - - auto to_tag_set = [](const std::vector &tags, bool case_sensitive) -> std::set { - if (case_sensitive) { - return {tags.begin(), tags.end()}; - } else { - std::set res; - std::transform(tags.begin(), tags.end(), std::inserter(res, res.begin()), util::ToLower); - return res; - } - }; - - std::set tags_to_delete = to_tag_set(original_tags, tag->case_sensitive); - std::set tags_to_add = to_tag_set(current_tags, tag->case_sensitive); - - for (auto it = tags_to_delete.begin(); it != tags_to_delete.end();) { - if (auto jt = tags_to_add.find(*it); jt != tags_to_add.end()) { - it = tags_to_delete.erase(it); - tags_to_add.erase(jt); - } else { - ++it; - } - } + auto batch = storage->GetWriteBatchBase(); + auto cf_handle = storage->GetCFHandle(ColumnFamilyID::Search); - if (tags_to_add.empty() && tags_to_delete.empty()) { - // no change, skip index updating - return Status::OK(); - } + for (const auto &tag : tags_to_delete) { + auto index_key = search_key.ConstructTagFieldData(tag, key); - auto batch = storage->GetWriteBatchBase(); - auto cf_handle = storage->GetCFHandle(ColumnFamilyID::Search); + batch->Delete(cf_handle, index_key); + } - for (const auto &tag : tags_to_delete) { - auto sub_key = ConstructTagFieldSubkey(field, tag, key); - auto index_key = InternalKey(ns_key, sub_key, info->metadata.version, storage->IsSlotIdEncoded()); + for (const auto &tag : tags_to_add) { + auto index_key = search_key.ConstructTagFieldData(tag, key); - batch->Delete(cf_handle, index_key.Encode()); - } + batch->Put(cf_handle, index_key, Slice()); + } - for (const auto &tag : tags_to_add) { - auto sub_key = ConstructTagFieldSubkey(field, tag, key); - auto index_key = InternalKey(ns_key, sub_key, info->metadata.version, storage->IsSlotIdEncoded()); + auto s = storage->Write(storage->DefaultWriteOptions(), batch->GetWriteBatch()); + if (!s.ok()) return {Status::NotOK, s.ToString()}; + return Status::OK(); +} - batch->Put(cf_handle, index_key.Encode(), Slice()); - } +Status IndexUpdater::UpdateNumericIndex(std::string_view key, std::string_view original, std::string_view current, + const SearchKey &search_key, const NumericFieldMetadata *num) const { + auto *storage = indexer->storage; + auto batch = storage->GetWriteBatchBase(); + auto cf_handle = storage->GetCFHandle(ColumnFamilyID::Search); - auto s = storage->Write(storage->DefaultWriteOptions(), batch->GetWriteBatch()); - if (!s.ok()) return {Status::NotOK, s.ToString()}; - } else if (auto numeric [[maybe_unused]] = dynamic_cast(metadata)) { - auto batch = storage->GetWriteBatchBase(); - auto cf_handle = storage->GetCFHandle(ColumnFamilyID::Search); + if (!original.empty()) { + auto original_num = GET_OR_RET(ParseFloat(std::string(original.begin(), original.end()))); + auto index_key = search_key.ConstructNumericFieldData(original_num, key); - if (!original.empty()) { - auto original_num = GET_OR_RET(ParseFloat(std::string(original.begin(), original.end()))); - auto sub_key = ConstructNumericFieldSubkey(field, original_num, key); - auto index_key = InternalKey(ns_key, sub_key, info->metadata.version, storage->IsSlotIdEncoded()); + batch->Delete(cf_handle, index_key); + } - batch->Delete(cf_handle, index_key.Encode()); - } + if (!current.empty()) { + auto current_num = GET_OR_RET(ParseFloat(std::string(current.begin(), current.end()))); + auto index_key = search_key.ConstructNumericFieldData(current_num, key); - if (!current.empty()) { - auto current_num = GET_OR_RET(ParseFloat(std::string(current.begin(), current.end()))); - auto sub_key = ConstructNumericFieldSubkey(field, current_num, key); - auto index_key = InternalKey(ns_key, sub_key, info->metadata.version, storage->IsSlotIdEncoded()); + batch->Put(cf_handle, index_key, Slice()); + } - batch->Put(cf_handle, index_key.Encode(), Slice()); - } + auto s = storage->Write(storage->DefaultWriteOptions(), batch->GetWriteBatch()); + if (!s.ok()) return {Status::NotOK, s.ToString()}; + return Status::OK(); +} - auto s = storage->Write(storage->DefaultWriteOptions(), batch->GetWriteBatch()); - if (!s.ok()) return {Status::NotOK, s.ToString()}; +Status IndexUpdater::UpdateIndex(const std::string &field, std::string_view key, std::string_view original, + std::string_view current, const std::string &ns) const { + if (original == current) { + // the value of this field is unchanged, no need to update + return Status::OK(); + } + + auto iter = info->fields.find(field); + if (iter == info->fields.end()) { + return {Status::NotOK, "No such field to do index updating"}; + } + + auto *metadata = iter->second.metadata.get(); + SearchKey search_key(ns, info->name, field); + if (auto tag = dynamic_cast(metadata)) { + GET_OR_RET(UpdateTagIndex(key, original, current, search_key, tag)); + } else if (auto numeric [[maybe_unused]] = dynamic_cast(metadata)) { + GET_OR_RET(UpdateNumericIndex(key, original, current, search_key, numeric)); } else { return {Status::NotOK, "Unexpected field type"}; } diff --git a/src/search/indexer.h b/src/search/indexer.h index dc1f03e0bdc..d6bf37def42 100644 --- a/src/search/indexer.h +++ b/src/search/indexer.h @@ -56,7 +56,7 @@ struct FieldValueRetriever { using Variant = std::variant; Variant db; - static StatusOr Create(SearchOnDataType type, std::string_view key, engine::Storage *storage, + static StatusOr Create(IndexOnDataType type, std::string_view key, engine::Storage *storage, const std::string &ns); explicit FieldValueRetriever(Hash hash, HashMetadata metadata, std::string_view key) @@ -79,6 +79,11 @@ struct IndexUpdater { Status UpdateIndex(const std::string &field, std::string_view key, std::string_view original, std::string_view current, const std::string &ns) const; Status Update(const FieldValues &original, std::string_view key, const std::string &ns) const; + + Status UpdateTagIndex(std::string_view key, std::string_view original, std::string_view current, + const SearchKey &search_key, const TagFieldMetadata *tag) const; + Status UpdateNumericIndex(std::string_view key, std::string_view original, std::string_view current, + const SearchKey &search_key, const NumericFieldMetadata *num) const; }; struct GlobalIndexer { diff --git a/src/search/ir_sema_checker.h b/src/search/ir_sema_checker.h index 170e646fb09..5e1006216ee 100644 --- a/src/search/ir_sema_checker.h +++ b/src/search/ir_sema_checker.h @@ -75,7 +75,7 @@ struct SemaChecker { } else if (auto v = dynamic_cast(node)) { if (auto iter = current_index->fields.find(v->field->name); iter == current_index->fields.end()) { return {Status::NotOK, fmt::format("field `{}` not found in index `{}`", v->field->name)}; - } else if (auto meta = iter->second.MetadataAs(); !meta) { + } else if (auto meta = iter->second.MetadataAs(); !meta) { return {Status::NotOK, fmt::format("field `{}` is not a tag field", v->field->name)}; } else { v->field->info = &iter->second; @@ -91,7 +91,7 @@ struct SemaChecker { } else if (auto v = dynamic_cast(node)) { if (auto iter = current_index->fields.find(v->field->name); iter == current_index->fields.end()) { return {Status::NotOK, fmt::format("field `{}` not found in index `{}`", v->field->name, current_index->name)}; - } else if (!iter->second.MetadataAs()) { + } else if (!iter->second.MetadataAs()) { return {Status::NotOK, fmt::format("field `{}` is not a numeric field", v->field->name)}; } else { v->field->info = &iter->second; diff --git a/src/search/passes/index_selection.h b/src/search/passes/index_selection.h index 4ced79727b2..e60287d4d01 100644 --- a/src/search/passes/index_selection.h +++ b/src/search/passes/index_selection.h @@ -72,7 +72,7 @@ struct IndexSelection : Visitor { bool HasGoodOrder() const { return order && order->field->info->HasIndex(); } std::unique_ptr GenerateScanFromOrder() const { - if (order->field->info->MetadataAs()) { + if (order->field->info->MetadataAs()) { return std::make_unique(order->field->CloneAs(), Interval::Full(), order->order); } else { CHECK(false) << "current only numeric field is supported for ordering"; diff --git a/src/search/search_encoding.h b/src/search/search_encoding.h index 32f244ca237..3819fd9bc26 100644 --- a/src/search/search_encoding.h +++ b/src/search/search_encoding.h @@ -23,26 +23,127 @@ #include #include +#include + namespace redis { +enum class IndexOnDataType : uint8_t { + HASH = kRedisHash, + JSON = kRedisJson, +}; + inline constexpr auto kErrorInsufficientLength = "insufficient length while decoding metadata"; +class IndexMetadata { + public: + uint8_t flag = 0; // all reserved + IndexOnDataType on_data_type; + + void Encode(std::string *dst) const { + PutFixed8(dst, flag); + PutFixed8(dst, uint8_t(on_data_type)); + } + + rocksdb::Status Decode(Slice *input) { + if (!GetFixed8(input, &flag)) { + return rocksdb::Status::InvalidArgument(kErrorInsufficientLength); + } + + if (!GetFixed8(input, reinterpret_cast(&on_data_type))) { + return rocksdb::Status::InvalidArgument(kErrorInsufficientLength); + } + + return rocksdb::Status::OK(); + } +}; + enum class SearchSubkeyType : uint8_t { - // search global metadata + INDEX_META = 0, + PREFIXES = 1, - // field metadata for different types - TAG_FIELD_META = 64 + 1, - NUMERIC_FIELD_META = 64 + 2, + // field metadata + FIELD_META = 2, + + // field indexing data + FIELD = 3, - // field indexing for different types - TAG_FIELD = 128 + 1, - NUMERIC_FIELD = 128 + 2, + // field alias + FIELD_ALIAS = 4, }; -inline std::string ConstructSearchPrefixesSubkey() { return {(char)SearchSubkeyType::PREFIXES}; } +enum class IndexFieldType : uint8_t { + TAG = 1, + + NUMERIC = 2, +}; + +struct SearchKey { + std::string_view ns; + std::string_view index; + std::string_view field; + + SearchKey(std::string_view ns, std::string_view index) : ns(ns), index(index) {} + SearchKey(std::string_view ns, std::string_view index, std::string_view field) : ns(ns), index(index), field(field) {} + + void PutNamespace(std::string *dst) const { + PutFixed8(dst, ns.size()); + dst->append(ns); + } + + static void PutType(std::string *dst, SearchSubkeyType type) { PutFixed8(dst, uint8_t(type)); } + + void PutIndex(std::string *dst) const { PutSizedString(dst, index); } + + std::string ConstructIndexMeta() const { + std::string dst; + PutNamespace(&dst); + PutType(&dst, SearchSubkeyType::INDEX_META); + PutIndex(&dst); + return dst; + } + + std::string ConstructIndexPrefixes() const { + std::string dst; + PutNamespace(&dst); + PutType(&dst, SearchSubkeyType::PREFIXES); + PutIndex(&dst); + return dst; + } + + std::string ConstructFieldMeta() const { + std::string dst; + PutNamespace(&dst); + PutType(&dst, SearchSubkeyType::FIELD_META); + PutIndex(&dst); + PutSizedString(&dst, field); + return dst; + } + + std::string ConstructTagFieldData(std::string_view tag, std::string_view key) const { + std::string dst; + PutNamespace(&dst); + PutType(&dst, SearchSubkeyType::FIELD); + PutIndex(&dst); + PutSizedString(&dst, field); + PutSizedString(&dst, tag); + PutSizedString(&dst, key); + return dst; + } + + std::string ConstructNumericFieldData(double num, std::string_view key) const { + std::string dst; + PutNamespace(&dst); + PutType(&dst, SearchSubkeyType::FIELD); + PutIndex(&dst); + PutSizedString(&dst, field); + PutDouble(&dst, num); + PutSizedString(&dst, key); + return dst; + } +}; -struct SearchPrefixesMetadata { +struct IndexPrefixes { std::vector prefixes; static inline const std::string all[] = {""}; @@ -75,15 +176,21 @@ struct SearchPrefixesMetadata { } }; -struct SearchFieldMetadata { +struct IndexFieldMetadata { bool noindex = false; + IndexFieldType type; - // flag: - uint8_t MakeFlag() const { return noindex; } + // flag: + uint8_t MakeFlag() const { return noindex | (uint8_t)type << 1; } - void DecodeFlag(uint8_t flag) { noindex = flag & 1; } + void DecodeFlag(uint8_t flag) { + noindex = flag & 1; + type = DecodeType(flag); + } + + static IndexFieldType DecodeType(uint8_t flag) { return IndexFieldType(flag >> 1); } - virtual ~SearchFieldMetadata() = default; + virtual ~IndexFieldMetadata() = default; virtual void Encode(std::string *dst) const { PutFixed8(dst, MakeFlag()); } @@ -96,26 +203,24 @@ struct SearchFieldMetadata { DecodeFlag(flag); return rocksdb::Status::OK(); } -}; -inline std::string ConstructTagFieldMetadataSubkey(std::string_view field_name) { - std::string res = {(char)SearchSubkeyType::TAG_FIELD_META}; - res.append(field_name); - return res; -} + virtual bool IsSortable() const { return false; } + + static inline rocksdb::Status Decode(Slice *input, std::unique_ptr &ptr); +}; -struct SearchTagFieldMetadata : SearchFieldMetadata { +struct TagFieldMetadata : IndexFieldMetadata { char separator = ','; bool case_sensitive = false; void Encode(std::string *dst) const override { - SearchFieldMetadata::Encode(dst); + IndexFieldMetadata::Encode(dst); PutFixed8(dst, separator); PutFixed8(dst, case_sensitive); } rocksdb::Status Decode(Slice *input) override { - if (auto s = SearchFieldMetadata::Decode(input); !s.ok()) { + if (auto s = IndexFieldMetadata::Decode(input); !s.ok()) { return s; } @@ -129,30 +234,27 @@ struct SearchTagFieldMetadata : SearchFieldMetadata { } }; -inline std::string ConstructNumericFieldMetadataSubkey(std::string_view field_name) { - std::string res = {(char)SearchSubkeyType::NUMERIC_FIELD_META}; - res.append(field_name); - return res; -} - -struct SearchSortableFieldMetadata : SearchFieldMetadata {}; +struct NumericFieldMetadata : IndexFieldMetadata { + bool IsSortable() const override { return true; } +}; -struct SearchNumericFieldMetadata : SearchSortableFieldMetadata {}; +inline rocksdb::Status IndexFieldMetadata::Decode(Slice *input, std::unique_ptr &ptr) { + if (input->size() < 1) { + return rocksdb::Status::Corruption(kErrorInsufficientLength); + } -inline std::string ConstructTagFieldSubkey(std::string_view field_name, std::string_view tag, std::string_view key) { - std::string res = {(char)SearchSubkeyType::TAG_FIELD}; - PutSizedString(&res, field_name); - PutSizedString(&res, tag); - PutSizedString(&res, key); - return res; -} + switch (DecodeType((*input)[0])) { + case IndexFieldType::TAG: + ptr = std::make_unique(); + break; + case IndexFieldType::NUMERIC: + ptr = std::make_unique(); + break; + default: + return rocksdb::Status::Corruption("encountered unknown field type"); + } -inline std::string ConstructNumericFieldSubkey(std::string_view field_name, double number, std::string_view key) { - std::string res = {(char)SearchSubkeyType::NUMERIC_FIELD}; - PutSizedString(&res, field_name); - PutDouble(&res, number); - PutSizedString(&res, key); - return res; + return ptr->Decode(input); } } // namespace redis diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc index 5e872af0eda..e44b39cad7c 100644 --- a/src/storage/redis_metadata.cc +++ b/src/storage/redis_metadata.cc @@ -96,6 +96,7 @@ std::string InternalKey::Encode() const { } bool InternalKey::operator==(const InternalKey &that) const { + if (namespace_ != this->namespace_) return false; if (key_ != that.key_) return false; if (sub_key_ != that.sub_key_) return false; return version_ == that.version_; @@ -471,21 +472,3 @@ rocksdb::Status JsonMetadata::Decode(Slice *input) { return rocksdb::Status::OK(); } - -void SearchMetadata::Encode(std::string *dst) const { - Metadata::Encode(dst); - - PutFixed8(dst, uint8_t(on_data_type)); -} - -rocksdb::Status SearchMetadata::Decode(Slice *input) { - if (auto s = Metadata::Decode(input); !s.ok()) { - return s; - } - - if (!GetFixed8(input, reinterpret_cast(&on_data_type))) { - return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); - } - - return rocksdb::Status::OK(); -} diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index 531a880355b..68f36b2c994 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -49,7 +49,6 @@ enum RedisType : uint8_t { kRedisStream = 8, kRedisBloomFilter = 9, kRedisJson = 10, - kRedisSearch = 11, }; struct RedisTypes { @@ -314,18 +313,3 @@ class JsonMetadata : public Metadata { void Encode(std::string *dst) const override; rocksdb::Status Decode(Slice *input) override; }; - -enum class SearchOnDataType : uint8_t { - HASH = kRedisHash, - JSON = kRedisJson, -}; - -class SearchMetadata : public Metadata { - public: - SearchOnDataType on_data_type; - - explicit SearchMetadata(bool generate_version = true) : Metadata(kRedisSearch, generate_version) {} - - void Encode(std::string *dst) const override; - rocksdb::Status Decode(Slice *input) override; -}; diff --git a/tests/cppunit/indexer_test.cc b/tests/cppunit/indexer_test.cc index 4ab41b3a5b9..1377989291d 100644 --- a/tests/cppunit/indexer_test.cc +++ b/tests/cppunit/indexer_test.cc @@ -36,24 +36,24 @@ struct IndexerTest : TestBase { std::string ns = "index_test"; IndexerTest() : indexer(storage_.get()) { - SearchMetadata hash_field_meta(false); - hash_field_meta.on_data_type = SearchOnDataType::HASH; + redis::IndexMetadata hash_field_meta; + hash_field_meta.on_data_type = redis::IndexOnDataType::HASH; auto hash_info = std::make_unique("hashtest", hash_field_meta); - hash_info->Add(kqir::FieldInfo("x", std::make_unique())); - hash_info->Add(kqir::FieldInfo("y", std::make_unique())); + hash_info->Add(kqir::FieldInfo("x", std::make_unique())); + hash_info->Add(kqir::FieldInfo("y", std::make_unique())); hash_info->prefixes.prefixes.emplace_back("idxtesthash"); map.emplace("hashtest", std::move(hash_info)); redis::IndexUpdater hash_updater{map.at("hashtest").get()}; - SearchMetadata json_field_meta(false); - json_field_meta.on_data_type = SearchOnDataType::JSON; + redis::IndexMetadata json_field_meta; + json_field_meta.on_data_type = redis::IndexOnDataType::JSON; auto json_info = std::make_unique("jsontest", json_field_meta); - json_info->Add(kqir::FieldInfo("$.x", std::make_unique())); - json_info->Add(kqir::FieldInfo("$.y", std::make_unique())); + json_info->Add(kqir::FieldInfo("$.x", std::make_unique())); + json_info->Add(kqir::FieldInfo("$.y", std::make_unique())); json_info->prefixes.prefixes.emplace_back("idxtestjson"); map.emplace("jsontest", std::move(json_info)); @@ -90,28 +90,22 @@ TEST_F(IndexerTest, HashTag) { auto s2 = indexer.Update(*s, key1, ns); ASSERT_TRUE(s2); - auto subkey = redis::ConstructTagFieldSubkey("x", "food", key1); - auto nskey = ComposeNamespaceKey(ns, idxname, false); - auto key = InternalKey(nskey, subkey, 0, false); + auto key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("food", key1); std::string val; - auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("x", "kitchen", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("kitchen", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("x", "beauty", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("beauty", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); } @@ -131,43 +125,33 @@ TEST_F(IndexerTest, HashTag) { auto s2 = indexer.Update(*s, key1, ns); ASSERT_TRUE(s2); - auto subkey = redis::ConstructTagFieldSubkey("x", "food", key1); - auto nskey = ComposeNamespaceKey(ns, idxname, false); - auto key = InternalKey(nskey, subkey, 0, false); + auto key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("food", key1); std::string val; - auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("x", "clothing", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("clothing", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("x", "sport", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("sport", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("x", "kitchen", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("kitchen", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.IsNotFound()); - subkey = redis::ConstructTagFieldSubkey("x", "beauty", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "x").ConstructTagFieldData("beauty", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.IsNotFound()); } } @@ -196,28 +180,22 @@ TEST_F(IndexerTest, JsonTag) { auto s2 = indexer.Update(*s, key1, ns); ASSERT_TRUE(s2); - auto subkey = redis::ConstructTagFieldSubkey("$.x", "food", key1); - auto nskey = ComposeNamespaceKey(ns, idxname, false); - auto key = InternalKey(nskey, subkey, 0, false); + auto key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("food", key1); std::string val; - auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("$.x", "kitchen", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("kitchen", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("$.x", "beauty", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("beauty", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); } @@ -235,43 +213,33 @@ TEST_F(IndexerTest, JsonTag) { auto s2 = indexer.Update(*s, key1, ns); ASSERT_TRUE(s2); - auto subkey = redis::ConstructTagFieldSubkey("$.x", "food", key1); - auto nskey = ComposeNamespaceKey(ns, idxname, false); - auto key = InternalKey(nskey, subkey, 0, false); + auto key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("food", key1); std::string val; - auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + auto s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("$.x", "clothing", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("clothing", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("$.x", "sport", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("sport", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.ok()); ASSERT_EQ(val, ""); - subkey = redis::ConstructTagFieldSubkey("$.x", "kitchen", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("kitchen", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.IsNotFound()); - subkey = redis::ConstructTagFieldSubkey("$.x", "beauty", key1); - nskey = ComposeNamespaceKey(ns, idxname, false); - key = InternalKey(nskey, subkey, 0, false); + key = redis::SearchKey(ns, idxname, "$.x").ConstructTagFieldData("beauty", key1); - s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key.Encode(), &val); + s3 = storage_->Get(storage_->DefaultMultiGetOptions(), cfhandler, key, &val); ASSERT_TRUE(s3.IsNotFound()); } } diff --git a/tests/cppunit/ir_dot_dumper_test.cc b/tests/cppunit/ir_dot_dumper_test.cc index 1615b3ca031..d616f290d83 100644 --- a/tests/cppunit/ir_dot_dumper_test.cc +++ b/tests/cppunit/ir_dot_dumper_test.cc @@ -27,6 +27,7 @@ #include "search/ir_plan.h" #include "search/ir_sema_checker.h" #include "search/passes/manager.h" +#include "search/search_encoding.h" #include "search/sql_transformer.h" using namespace kqir; @@ -64,14 +65,14 @@ static auto ParseS(SemaChecker& sc, const std::string& in) { } static IndexMap MakeIndexMap() { - auto f1 = FieldInfo("t1", std::make_unique()); - auto f2 = FieldInfo("t2", std::make_unique()); + auto f1 = FieldInfo("t1", std::make_unique()); + auto f2 = FieldInfo("t2", std::make_unique()); f2.metadata->noindex = true; - auto f3 = FieldInfo("n1", std::make_unique()); - auto f4 = FieldInfo("n2", std::make_unique()); - auto f5 = FieldInfo("n3", std::make_unique()); + auto f3 = FieldInfo("n1", std::make_unique()); + auto f4 = FieldInfo("n2", std::make_unique()); + auto f5 = FieldInfo("n3", std::make_unique()); f5.metadata->noindex = true; - auto ia = std::make_unique("ia", SearchMetadata()); + auto ia = std::make_unique("ia", redis::IndexMetadata()); ia->Add(std::move(f1)); ia->Add(std::move(f2)); ia->Add(std::move(f3)); diff --git a/tests/cppunit/ir_pass_test.cc b/tests/cppunit/ir_pass_test.cc index 6188bb49f44..76318d79af2 100644 --- a/tests/cppunit/ir_pass_test.cc +++ b/tests/cppunit/ir_pass_test.cc @@ -169,14 +169,14 @@ TEST(IRPassTest, IntervalAnalysis) { } static IndexMap MakeIndexMap() { - auto f1 = FieldInfo("t1", std::make_unique()); - auto f2 = FieldInfo("t2", std::make_unique()); + auto f1 = FieldInfo("t1", std::make_unique()); + auto f2 = FieldInfo("t2", std::make_unique()); f2.metadata->noindex = true; - auto f3 = FieldInfo("n1", std::make_unique()); - auto f4 = FieldInfo("n2", std::make_unique()); - auto f5 = FieldInfo("n3", std::make_unique()); + auto f3 = FieldInfo("n1", std::make_unique()); + auto f4 = FieldInfo("n2", std::make_unique()); + auto f5 = FieldInfo("n3", std::make_unique()); f5.metadata->noindex = true; - auto ia = std::make_unique("ia", SearchMetadata()); + auto ia = std::make_unique("ia", redis::IndexMetadata()); ia->Add(std::move(f1)); ia->Add(std::move(f2)); ia->Add(std::move(f3)); diff --git a/tests/cppunit/ir_sema_checker_test.cc b/tests/cppunit/ir_sema_checker_test.cc index 678a0a0fa33..a12beea7363 100644 --- a/tests/cppunit/ir_sema_checker_test.cc +++ b/tests/cppunit/ir_sema_checker_test.cc @@ -35,10 +35,10 @@ using namespace kqir; static auto Parse(const std::string& in) { return sql::ParseToIR(peg::string_input(in, "test")); } static IndexMap MakeIndexMap() { - auto f1 = FieldInfo("f1", std::make_unique()); - auto f2 = FieldInfo("f2", std::make_unique()); - auto f3 = FieldInfo("f3", std::make_unique()); - auto ia = std::make_unique("ia", SearchMetadata()); + auto f1 = FieldInfo("f1", std::make_unique()); + auto f2 = FieldInfo("f2", std::make_unique()); + auto f3 = FieldInfo("f3", std::make_unique()); + auto ia = std::make_unique("ia", redis::IndexMetadata()); ia->Add(std::move(f1)); ia->Add(std::move(f2)); ia->Add(std::move(f3)); diff --git a/tests/cppunit/plan_executor_test.cc b/tests/cppunit/plan_executor_test.cc index 0b225fc7a1d..bad978d03ab 100644 --- a/tests/cppunit/plan_executor_test.cc +++ b/tests/cppunit/plan_executor_test.cc @@ -37,12 +37,12 @@ using namespace kqir; static auto exe_end = ExecutorNode::Result(ExecutorNode::end); static IndexMap MakeIndexMap() { - auto f1 = FieldInfo("f1", std::make_unique()); - auto f2 = FieldInfo("f2", std::make_unique()); - auto f3 = FieldInfo("f3", std::make_unique()); - auto ia = std::make_unique("ia", SearchMetadata()); + auto f1 = FieldInfo("f1", std::make_unique()); + auto f2 = FieldInfo("f2", std::make_unique()); + auto f3 = FieldInfo("f3", std::make_unique()); + auto ia = std::make_unique("ia", redis::IndexMetadata()); ia->ns = "search_ns"; - ia->metadata.on_data_type = SearchOnDataType::JSON; + ia->metadata.on_data_type = redis::IndexOnDataType::JSON; ia->prefixes.prefixes.emplace_back("test2:"); ia->prefixes.prefixes.emplace_back("test4:"); ia->Add(std::move(f1));