From fdc8799ca73544cdd43e1994779a72f28f786a93 Mon Sep 17 00:00:00 2001 From: reindexer-bot <@> Date: Fri, 2 Feb 2024 13:26:54 +0000 Subject: [PATCH] Merge branch 'develop' into 1661_482_merge_queryEntries --- .../test/test_storage_compatibility.sh | 195 ++++ cpp_src/core/keyvalue/fast_hash_set_variant.h | 16 + cpp_src/core/keyvalue/variant.h | 25 +- cpp_src/core/nsselecter/nsselecter.cc | 8 +- cpp_src/core/nsselecter/querypreprocessor.cc | 879 ++++++++++++++++-- cpp_src/core/nsselecter/querypreprocessor.h | 43 +- .../nsselecter/selectiteratorcontainer.cc | 12 + .../core/nsselecter/selectiteratorcontainer.h | 2 + cpp_src/core/query/queryentry.cc | 29 +- cpp_src/core/query/queryentry.h | 4 + cpp_src/estl/forward_like.h | 26 + cpp_src/gtests/tests/fixtures/queries_api.h | 56 +- cpp_src/gtests/tests/unit/ft/ft_generic.cc | 4 +- .../gtests/tests/unit/selector_plan_test.cc | 126 ++- fulltext.md | 6 +- test/queries_test.go | 22 +- 16 files changed, 1253 insertions(+), 200 deletions(-) create mode 100755 cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh create mode 100644 cpp_src/core/keyvalue/fast_hash_set_variant.h create mode 100644 cpp_src/estl/forward_like.h diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100755 index 000000000..d189d3841 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +echo "## choose latest release rpm file" +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 3) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 4) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/core/keyvalue/fast_hash_set_variant.h b/cpp_src/core/keyvalue/fast_hash_set_variant.h new file mode 100644 index 000000000..bf489ea1f --- /dev/null +++ b/cpp_src/core/keyvalue/fast_hash_set_variant.h @@ -0,0 +1,16 @@ +#pragma once + +#include "estl/fast_hash_set.h" +#include "variant.h" + +namespace reindexer { + +class fast_hash_set_variant : public fast_hash_set, Variant::EqualTo, Variant::Less> { + using Base = fast_hash_set, Variant::EqualTo, Variant::Less>; + +public: + fast_hash_set_variant(const CollateOpts& collate) + : Base{16, std::hash{}, Variant::EqualTo{collate}, typename Base::allocator_type{}, Variant::Less{collate}} {} +}; + +} // namespace reindexer diff --git a/cpp_src/core/keyvalue/variant.h b/cpp_src/core/keyvalue/variant.h index 6fe5b532a..c39adc261 100644 --- a/cpp_src/core/keyvalue/variant.h +++ b/cpp_src/core/keyvalue/variant.h @@ -67,7 +67,7 @@ class Variant { template Variant(const std::tuple &); - Variant &operator=(Variant &&other) &noexcept { + Variant &operator=(Variant &&other) & noexcept { if (this == &other) return *this; if (isUuid()) { if (other.isUuid()) { @@ -119,6 +119,7 @@ class Variant { bool operator<(const Variant &other) const { return Compare(other) < 0; } bool operator>(const Variant &other) const { return Compare(other) > 0; } bool operator>=(const Variant &other) const { return Compare(other) >= 0; } + bool operator<=(const Variant &other) const { return Compare(other) <= 0; } int Compare(const Variant &other, const CollateOpts &collateOpts = CollateOpts()) const; template @@ -147,6 +148,24 @@ class Variant { template void Dump(T &os, CheckIsStringPrintable checkPrintableString = CheckIsStringPrintable::Yes) const; + class Less { + public: + Less(const CollateOpts &collate) noexcept : collate_{&collate} {} + [[nodiscard]] bool operator()(const Variant &lhs, const Variant &rhs) const { return lhs.Compare(rhs, *collate_) < 0; } + + private: + const CollateOpts *collate_; + }; + + class EqualTo { + public: + EqualTo(const CollateOpts &collate) noexcept : collate_{&collate} {} + [[nodiscard]] bool operator()(const Variant &lhs, const Variant &rhs) const { return lhs.Compare(rhs, *collate_) == 0; } + + private: + const CollateOpts *collate_; + }; + private: bool isUuid() const noexcept { return uuid_.isUuid != 0; } void convertToComposite(const PayloadType &, const FieldsSet &); @@ -223,11 +242,11 @@ class VariantArray : public h_vector { VariantArray() noexcept = default; explicit VariantArray(Point) noexcept; explicit operator Point() const; - VariantArray &MarkArray(bool v = true) &noexcept { + VariantArray &MarkArray(bool v = true) & noexcept { isArrayValue = v; return *this; } - VariantArray &&MarkArray(bool v = true) &&noexcept { + VariantArray &&MarkArray(bool v = true) && noexcept { isArrayValue = v; return std::move(*this); } diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 9f69f7b0d..9825a0e1d 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -94,6 +94,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte fnc_ = ctx.functions->AddNamespace(ctx.query, *ns_, ctx.nsid, isFt); } + qPreproc.ConvertWhereValues(); if (!ctx.skipIndexesLookup) { qPreproc.Reduce(isFt); } @@ -101,7 +102,6 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte qPreproc.CheckUniqueFtQuery(); qPreproc.ExcludeFtQuery(rdxCtx); } - qPreproc.ConvertWhereValues(); explain.AddPrepareTime(); @@ -252,7 +252,11 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte if (it.comparators_.size()) hasComparators = true; }); - if (!qres.HasIdsets()) { + if (qres.HasAlwaysFalse()) { + SelectKeyResult zeroScan; + zeroScan.emplace_back(0, 0); + qres.AppendFront(OpAnd, SelectIterator{std::move(zeroScan), false, "always_false", IteratorFieldKind::None, true}); + } else if (!qres.HasIdsets()) { SelectKeyResult scan; if (ctx.sortingContext.isOptimizationEnabled()) { auto it = ns_->indexes_[ctx.sortingContext.uncommitedIndex]->CreateIterator(); diff --git a/cpp_src/core/nsselecter/querypreprocessor.cc b/cpp_src/core/nsselecter/querypreprocessor.cc index 5eda4812a..a177b4483 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.cc +++ b/cpp_src/core/nsselecter/querypreprocessor.cc @@ -1,6 +1,7 @@ #include "querypreprocessor.h" #include "core/index/index.h" +#include "core/keyvalue/fast_hash_set_variant.h" #include "core/namespace/namespaceimpl.h" #include "core/nsselecter/joinedselector.h" #include "core/nsselecter/selectiteratorcontainer.h" @@ -129,20 +130,158 @@ void QueryPreprocessor::InjectConditionsFromJoins(JoinedSelectors &js, OnConditi } } +bool QueryPreprocessor::removeAlwaysFalse() { + const auto [deleted, changed] = removeAlwaysFalse(0, Size()); + return changed || deleted; +} + +std::pair QueryPreprocessor::removeAlwaysFalse(size_t begin, size_t end) { + size_t deleted = 0; + bool changed = false; + for (size_t i = begin; i < end - deleted;) { + if (IsSubTree(i)) { + const auto [d, ch] = removeAlwaysFalse(i + 1, Next(i)); + deleted += d; + changed = changed || ch; + i = Next(i); + } else if (Is(i)) { + switch (GetOperation(i)) { + case OpOr: + Erase(i, i + 1); + ++deleted; + break; + case OpNot: + SetValue(i, AlwaysTrue{}); + SetOperation(OpAnd, i); + changed = true; + ++i; + break; + case OpAnd: + if (i + 1 < end - deleted && GetOperation(i + 1) == OpOr) { + Erase(i, i + 1); + SetOperation(OpAnd, i); + ++deleted; + break; + } else { + Erase(i + 1, end - deleted); + Erase(begin, i); + return {end - begin - 1, false}; + } + } + } else { + i = Next(i); + } + } + return {deleted, changed}; +} + +bool QueryPreprocessor::removeAlwaysTrue() { + const auto [deleted, changed] = removeAlwaysTrue(0, Size()); + return changed || deleted; +} + +bool QueryPreprocessor::containsJoin(size_t n) noexcept { + return InvokeAppropriate( + n, [](const JoinQueryEntry &) noexcept { return true; }, [](const QueryEntry &) noexcept { return false; }, + [](const BetweenFieldsQueryEntry &) noexcept { return false; }, [](const AlwaysTrue &) noexcept { return false; }, + [](const AlwaysFalse &) noexcept { return false; }, [](const SubQueryEntry &) noexcept { return false; }, + [](const SubQueryFieldEntry &) noexcept { return false; }, + [&](const QueryEntriesBracket &) noexcept { + for (size_t i = n, e = Next(n); i < e; ++i) { + if (Is(i)) { + return true; + } + } + return false; + }); +} + +std::pair QueryPreprocessor::removeAlwaysTrue(size_t begin, size_t end) { + size_t deleted = 0; + bool changed = false; + for (size_t i = begin, prev = begin; i < end - deleted;) { + if (IsSubTree(i)) { + const auto [d, ch] = removeAlwaysTrue(i + 1, Next(i)); + deleted += d; + if (Size(i) == 1) { + SetValue(i, AlwaysTrue{}); + changed = true; + } else { + prev = i; + i = Next(i); + changed = changed || ch; + } + } else if (Is(i)) { + switch (GetOperation(i)) { + case OpAnd: + if (i + 1 >= end - deleted || GetOperation(i + 1) != OpOr) { + Erase(i, i + 1); + ++deleted; + } else { + size_t n = i + 1; + while (n < end - deleted && GetOperation(n) == OpOr) { + if (containsJoin(n)) { + n = Next(n); + } else { + deleted += Size(n); + Erase(n, Next(n)); + } + } + } + break; + case OpNot: + SetValue(i, AlwaysFalse{}); + SetOperation(OpAnd, i); + changed = true; + prev = i; + ++i; + break; + case OpOr: { + size_t n = i; + size_t prevN = prev; + do { + assertrx_throw(prevN < n); + bool needMoveI = false; + if (!containsJoin(prevN)) { + if (GetOperation(prevN) != OpOr) { + SetOperation(OpAnd, n); + } + deleted += Size(prevN); + Erase(prevN, n); + needMoveI = (n == i); + } + n = prevN; + prevN = begin; + while (Next(prevN) < n) { + prevN = Next(prevN); + } + if (needMoveI) { + i = n; + prev = prevN; + } + } while (GetOperation(n) == OpOr); + } break; + } + } else { + prev = i; + i = Next(i); + } + } + return {deleted, changed}; +} + void QueryPreprocessor::Reduce(bool isFt) { bool changed; do { changed = removeBrackets(); changed = LookupQueryIndexes() || changed; + changed = removeAlwaysFalse() || changed; + changed = removeAlwaysTrue() || changed; if (!isFt) changed = SubstituteCompositeIndexes() || changed; } while (changed); } -bool QueryPreprocessor::removeBrackets() { - const size_t startSize = Size(); - removeBrackets(0, startSize); - return startSize != Size(); -} +bool QueryPreprocessor::removeBrackets() { return removeBrackets(0, Size()); } bool QueryPreprocessor::canRemoveBracket(size_t i) const { if (Size(i) < 2) { @@ -200,12 +339,12 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui size_t merged = 0; for (size_t src = srcBegin, nextSrc; src < srcEnd; src = nextSrc) { nextSrc = Next(src); - const bool changeDst = container_[src].InvokeAppropriate( - [](const SubQueryEntry &) -> bool { + const MergeResult mergeResult = container_[src].InvokeAppropriate( + [](const SubQueryEntry &) -> MergeResult { assertrx_throw(0); abort(); }, - [](const SubQueryFieldEntry &) -> bool { + [](const SubQueryFieldEntry &) -> MergeResult { assertrx_throw(0); abort(); }, @@ -214,7 +353,7 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui const size_t mergedInBracket = lookupQueryIndexes(dst + 1, src + 1, nextSrc); container_[dst].Value().Erase(mergedInBracket); merged += mergedInBracket; - return true; + return MergeResult::NotMerged; }, [&](QueryEntry &entry) { if (entry.IsFieldIndexed()) { @@ -226,10 +365,19 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui std::fill(iidx.begin() + oldSize, iidx.begin() + iidx.size(), 0); } auto &iidxRef = iidx[entry.IndexNo()]; - if (iidxRef > 0 && !ns_.indexes_[entry.IndexNo()]->Opts().IsArray()) { - if (mergeQueryEntries(iidxRef - 1, src)) { - ++merged; - return false; + const Index &index = *ns_.indexes_[entry.IndexNo()]; + const auto &indexOpts = index.Opts(); + if (iidxRef > 0 && !indexOpts.IsArray()) { + switch (index.IsOrdered() ? mergeQueryEntries(iidxRef - 1, src, indexOpts.collateOpts_) + : mergeQueryEntries(iidxRef - 1, src, indexOpts.collateOpts_)) { + case MergeResult::NotMerged: + break; + case MergeResult::Merged: + ++merged; + return MergeResult::Merged; + case MergeResult::Annihilated: + iidxRef = 0; + return MergeResult::Annihilated; } } else { assertrx_throw(dst < std::numeric_limits::max() - 1); @@ -238,25 +386,33 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui } } if (dst != src) container_[dst] = std::move(container_[src]); - return true; + return MergeResult::NotMerged; }, - [dst, src, this](JoinQueryEntry &) { + [dst, src, this](const JoinQueryEntry &) { if (dst != src) container_[dst] = std::move(container_[src]); - return true; + return MergeResult::NotMerged; }, - [dst, src, this](BetweenFieldsQueryEntry &) { + [dst, src, this](const BetweenFieldsQueryEntry &) { if (dst != src) container_[dst] = std::move(container_[src]); - return true; + return MergeResult::NotMerged; }, - [dst, src, this](AlwaysFalse &) { + [dst, src, this](const AlwaysFalse &) { if (dst != src) container_[dst] = std::move(container_[src]); - return true; + return MergeResult::NotMerged; }, - [dst, src, this](AlwaysTrue &) { + [dst, src, this](const AlwaysTrue &) { if (dst != src) container_[dst] = std::move(container_[src]); - return true; + return MergeResult::NotMerged; }); - if (changeDst) dst = Next(dst); + switch (mergeResult) { + case MergeResult::NotMerged: + dst = Next(dst); + break; + case MergeResult::Merged: + break; + case MergeResult::Annihilated: + return merged + srcEnd - src; + } } return merged; } @@ -502,64 +658,645 @@ void QueryPreprocessor::findMaxIndex(QueryEntries::const_iterator begin, QueryEn } } -bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { - QueryEntry *lqe = &Get(lhs); - QueryEntry &rqe = Get(rhs); - if ((lqe->Condition() == CondEq || lqe->Condition() == CondSet) && (rqe.Condition() == CondEq || rqe.Condition() == CondSet)) { - // intersect 2 queryentries on the same index - if rx_unlikely (lqe->Values().empty()) { - return true; +constexpr size_t kMinArraySizeToUseHashSet = 250; +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesSetSet(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, + const CollateOpts &collate) { + // intersect 2 queryentries on the same index + if rx_unlikely (lqe.Values().empty() || rqe.Values().empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + auto &&[first, second] = lqe.Values().size() < rqe.Values().size() ? std::make_pair(std::move(lqe).Values(), std::move(rqe).Values()) + : std::make_pair(std::move(rqe).Values(), std::move(lqe).Values()); + + if (first.size() == 1) { + const Variant &firstV = first[0]; + const Variant::EqualTo equalTo{collate}; + for (const Variant &secondV : second) { + if (equalTo(firstV, secondV)) { + lqe.SetCondAndValues(CondEq, VariantArray{std::move(first[0])}); + lqe.Distinct(distinct); + return MergeResult::Merged; + } } - const bool distinct = lqe->Distinct() || rqe.Distinct(); + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else { VariantArray setValues; - if (rx_likely(!rqe.Values().empty())) { - convertWhereValues(*lqe); - convertWhereValues(rqe); - auto &&[first, second] = lqe->Values().size() < rqe.Values().size() - ? std::make_pair(std::move(*lqe).Values(), std::move(rqe).Values()) - : std::make_pair(std::move(rqe).Values(), std::move(*lqe).Values()); - - setValues.reserve(first.size()); - constexpr size_t kMinArraySizeToUseHashSet = 250; - if (second.size() < kMinArraySizeToUseHashSet) { - // Intersect via binary search + sort for small vectors - boost::sort::pdqsort(first.begin(), first.end()); - for (auto &&v : second) { - if (std::binary_search(first.begin(), first.end(), v)) { - setValues.emplace_back(std::move(v)); - } + setValues.reserve(first.size()); + if (second.size() < kMinArraySizeToUseHashSet) { + // Intersect via binary search + sort for small vectors + boost::sort::pdqsort(first.begin(), first.end(), Variant::Less{collate}); + for (auto &&v : second) { + if (std::binary_search(first.begin(), first.end(), v, Variant::Less{collate})) { + setValues.emplace_back(std::move(v)); } - } else { - // Intersect via hash_set for large vectors - reindexer::fast_hash_set set; - set.reserve(first.size() * 2); - for (auto &&v : first) { - set.emplace(std::move(v)); - } - for (auto &&v : second) { - if (set.erase(v)) { - setValues.emplace_back(std::move(v)); - } + } + } else { + // Intersect via hash_set for large vectors + fast_hash_set_variant set{collate}; + set.reserve(first.size() * 2); + for (auto &&v : first) { + set.emplace(std::move(v)); + } + for (auto &&v : second) { + if (set.erase(v)) { + setValues.emplace_back(std::move(v)); } } } + if rx_unlikely (setValues.empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + lqe.SetCondAndValues(CondSet, std::move(setValues)); // NOLINT (bugprone-use-after-move) + lqe.Distinct(distinct); + return MergeResult::Merged; + } +} - lqe->SetCondAndValues(CondSet, std::move(setValues)); - lqe->Distinct(distinct); - return true; - } else if (rqe.Condition() == CondAny) { - if (!lqe->Distinct() && rqe.Distinct()) { - lqe->Distinct(true); +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesAllSetSet(QueryEntry &lqe, QueryEntry &rqe, bool distinct, + size_t position, const CollateOpts &collate) { + if rx_unlikely (lqe.Values().empty() || rqe.Values().empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + const Variant::EqualTo equalTo{collate}; + const Variant &lv = lqe.Values()[0]; + for (size_t i = 1, s = lqe.Values().size(); i < s; ++i) { + if (!equalTo(lv, lqe.Values()[i])) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; } - return true; - } else if (lqe->Condition() == CondAny) { - const bool distinct = lqe->Distinct() || rqe.Distinct(); - container_[lhs].SetValue(std::move(rqe)); - Get(lhs).Distinct(distinct); - return true; } + QueryEntry &dst = needSwitch == NeedSwitch::Yes ? rqe : lqe; + for (const Variant &rv : rqe.Values()) { + if (equalTo(lv, rv)) { + dst.Distinct(distinct); + dst.SetCondAndValues(CondEq, VariantArray{std::move(std::move(lqe).Values()[0])}); + return MergeResult::Merged; + } + } + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; +} - return false; +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesAllSetAllSet(QueryEntry &lqe, QueryEntry &rqe, bool distinct, + size_t position, const CollateOpts &collate) { + if rx_unlikely (lqe.Values().empty() || rqe.Values().empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + const Variant::EqualTo equalTo{collate}; + const Variant &lv = lqe.Values()[0]; + for (size_t i = 1, s = lqe.Values().size(); i < s; ++i) { + if (!equalTo(lv, lqe.Values()[i])) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + } + for (const Variant &rv : rqe.Values()) { + if (!equalTo(lv, rv)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + } + lqe.Distinct(distinct); + lqe.SetCondAndValues(CondEq, VariantArray{std::move(std::move(lqe).Values()[0])}); + return MergeResult::Merged; +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesAny(QueryEntry &lqe, QueryEntry &rqe, bool distinct) { + if constexpr (needSwitch == NeedSwitch::Yes) { + lqe = std::move(rqe); + } + lqe.Distinct(distinct); + return MergeResult::Merged; +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesSetNotSet(QueryEntry &lqe, QueryEntry &rqe, F filter, bool distinct, + size_t position) { + if rx_unlikely (lqe.Values().empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + { + auto updatableValues = lqe.UpdatableValues(); + VariantArray &values = updatableValues; + values.erase(std::remove_if(values.begin(), values.end(), filter), values.end()); + } + if rx_unlikely (lqe.Values().empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + if constexpr (mergeOrdered == MergeOrdered::No) { + lqe.Distinct(distinct); + if constexpr (needSwitch == NeedSwitch::Yes) { + rqe = std::move(lqe); + } + return MergeResult::Merged; + } else { + return MergeResult::NotMerged; + } +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesAllSetNotSet(QueryEntry &lqe, QueryEntry &rqe, F filter, bool distinct, + size_t position, const CollateOpts &collate) { + if rx_unlikely (lqe.Values().empty()) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + const Variant::EqualTo equalTo{collate}; + const Variant &lv = lqe.Values()[0]; + for (size_t i = 1, s = lqe.Values().size(); i < s; ++i) { + if (!equalTo(lv, lqe.Values()[i])) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + } + if (!filter(lv)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } + if constexpr (mergeOrdered == MergeOrdered::No) { + QueryEntry &dst = needSwitch == NeedSwitch::Yes ? rqe : lqe; + dst.Distinct(distinct); + dst.SetCondAndValues(CondEq, VariantArray{std::move(std::move(lqe).Values()[0])}); + return MergeResult::Merged; + } else { + lqe.SetCondAndValues(CondEq, VariantArray{std::move(std::move(lqe).Values()[0])}); + return MergeResult::NotMerged; + } +} + +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesLt(QueryEntry &lqe, QueryEntry &rqe, bool distinct, + const CollateOpts &collate) { + const Variant &lv = lqe.Values()[0]; + const Variant &rv = rqe.Values()[0]; + const Variant::Less less{collate}; + if (less(rv, lv)) { + lqe.SetCondAndValues(rqe.Condition(), std::move(rqe).Values()); // NOLINT (bugprone-use-after-move) + } else if (!less(lv, rv) && (lqe.Condition() != rqe.Condition())) { + lqe.SetCondAndValues(CondLt, std::move(rqe).Values()); + } + lqe.Distinct(distinct); + return MergeResult::Merged; +} + +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesGt(QueryEntry &lqe, QueryEntry &rqe, bool distinct, + const CollateOpts &collate) { + const Variant &lv = lqe.Values()[0]; + const Variant &rv = rqe.Values()[0]; + if (Variant::Less{collate}(lv, rv)) { + lqe.SetCondAndValues(rqe.Condition(), std::move(rqe).Values()); // NOLINT (bugprone-use-after-move) + } else if (Variant::EqualTo{collate}(lv, rv) && (lqe.Condition() != rqe.Condition())) { + lqe.SetCondAndValues(CondGt, std::move(rqe).Values()); + } + lqe.Distinct(distinct); + return MergeResult::Merged; +} + +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesLtGt(QueryEntry &lqe, QueryEntry &rqe, size_t position, + const CollateOpts &collate) { + const Variant &lv = lqe.Values()[0]; + const Variant &rv = rqe.Values()[0]; + if (Variant::Less{collate}(rv, lv)) { + return MergeResult::NotMerged; + } else { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesLeGe(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, + const CollateOpts &collate) { + const Variant &lv = lqe.Values()[0]; + const Variant &rv = rqe.Values()[0]; + const Variant::Less less{collate}; + QueryEntry &target = needSwitch == NeedSwitch::No ? lqe : rqe; + QueryEntry &source = needSwitch == NeedSwitch::No ? rqe : lqe; + if (less(lv, rv)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else if (less(rv, lv)) { + target.SetCondAndValues(CondRange, VariantArray{std::move(rqe).Values()[0], std::move(lqe).Values()[0]}); + } else { + target.SetCondAndValues(CondEq, std::move(source).Values()); + } + target.Distinct(distinct); + return MergeResult::Merged; +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesRangeLt(QueryEntry &range, QueryEntry <, bool distinct, + size_t position, const CollateOpts &collate) { + const Variant <V = lt.Values()[0]; + const Variant &rngL = range.Values()[0]; + const Variant &rngR = range.Values()[1]; + const Variant::Less less{collate}; + if (!less(rngL, ltV)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else if (less(rngR, ltV)) { + range.Distinct(distinct); + if constexpr (needSwitch == NeedSwitch::Yes) { + lt = std::move(range); + } + return MergeResult::Merged; + } else { + return MergeResult::NotMerged; + } +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesRangeGt(QueryEntry &range, QueryEntry >, bool distinct, + size_t position, const CollateOpts &collate) { + const Variant >V = gt.Values()[0]; + const Variant &rngL = range.Values()[0]; + const Variant &rngR = range.Values()[1]; + const Variant::Less less{collate}; + if (!less(gtV, rngR)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else if (less(gtV, rngL)) { + range.Distinct(distinct); + if constexpr (needSwitch == NeedSwitch::Yes) { + gt = std::move(range); + } + return MergeResult::Merged; + } else { + return MergeResult::NotMerged; + } +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesRangeLe(QueryEntry &range, QueryEntry &le, bool distinct, + size_t position, const CollateOpts &collate) { + const Variant &leV = le.Values()[0]; + const Variant &rngL = range.Values()[0]; + const Variant &rngR = range.Values()[1]; + const Variant::Less less{collate}; + QueryEntry &target = needSwitch == NeedSwitch::No ? range : le; + if (less(leV, rngL)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else if (Variant::EqualTo{collate}(leV, rngL)) { + target.SetCondAndValues(CondEq, std::move(le).Values()); + target.Distinct(distinct); + } else if (less(leV, rngR)) { + target.SetCondAndValues(CondRange, VariantArray{std::move(range).Values()[0], std::move(le).Values()[0]}); + target.Distinct(distinct); + } else { + range.Distinct(distinct); + if constexpr (needSwitch == NeedSwitch::Yes) { + le = std::move(range); + } + } + return MergeResult::Merged; +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesRangeGe(QueryEntry &range, QueryEntry &ge, bool distinct, + size_t position, const CollateOpts &collate) { + const Variant &geV = ge.Values()[0]; + const Variant &rngL = range.Values()[0]; + const Variant &rngR = range.Values()[1]; + const Variant::Less less{collate}; + QueryEntry &target = needSwitch == NeedSwitch::No ? range : ge; + if (less(rngR, geV)) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else if (Variant::EqualTo{collate}(geV, rngR)) { + target.SetCondAndValues(CondEq, std::move(ge).Values()); + target.Distinct(distinct); + } else if (less(rngL, geV)) { + target.SetCondAndValues(CondRange, VariantArray{std::move(ge).Values()[0], std::move(range).Values()[1]}); + target.Distinct(distinct); + } else { + range.Distinct(distinct); + if constexpr (needSwitch == NeedSwitch::Yes) { + ge = std::move(range); + } + } + return MergeResult::Merged; +} + +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesRange(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, + const CollateOpts &collate) { + const Variant::Less less{collate}; + QueryEntry &left = less(lqe.Values()[0], rqe.Values()[0]) ? rqe : lqe; + QueryEntry &right = less(rqe.Values()[1], lqe.Values()[1]) ? rqe : lqe; + if (less(right.Values()[1], left.Values()[0])) { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } else if (Variant::EqualTo{collate}(left.Values()[0], right.Values()[1])) { + lqe.SetCondAndValues(CondEq, VariantArray::Create(std::move(left).Values()[0])); + lqe.Distinct(distinct); + } else { + lqe.SetCondAndValues(CondRange, VariantArray::Create(std::move(left).Values()[0], std::move(right).Values()[1])); + lqe.Distinct(distinct); + } + return MergeResult::Merged; +} + +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntriesDWithin(QueryEntry &lqe, QueryEntry &rqe, bool distinct, + size_t position) { + Point lp, rp; + double ld, rd; + if (lqe.Values()[0].Type().Is()) { + lp = lqe.Values()[0].As(); + ld = lqe.Values()[1].As(); + } else { + lp = lqe.Values()[1].As(); + ld = lqe.Values()[0].As(); + } + if (rqe.Values()[0].Type().Is()) { + rp = rqe.Values()[0].As(); + rd = rqe.Values()[1].As(); + } else { + rp = rqe.Values()[1].As(); + rd = rqe.Values()[0].As(); + } + if (lp == rp) { + lqe.SetCondAndValues(CondDWithin, VariantArray::Create(lp, std::min(ld, rd))); + lqe.Distinct(distinct); + return MergeResult::Merged; + } else if (DWithin(lp, rp, ld + rd)) { + return MergeResult::NotMerged; + } else { + SetValue(position, AlwaysFalse{}); + return MergeResult::Annihilated; + } +} + +template +QueryPreprocessor::MergeResult QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs, const CollateOpts &collate) { + QueryEntry &lqe = Get(lhs); + QueryEntry &rqe = Get(rhs); + const bool distinct = lqe.Distinct() || rqe.Distinct(); + const Variant::Less less{collate}; + switch (lqe.Condition()) { + case CondEq: + case CondSet: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesSetSet(lqe, rqe, distinct, lhs, collate); + case CondAllSet: + return mergeQueryEntriesAllSetSet(rqe, lqe, distinct, lhs, collate); + case CondLt: + return mergeQueryEntriesSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return !less(v, rv); }, distinct, lhs); + case CondLe: + return mergeQueryEntriesSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return less(rv, v); }, distinct, lhs); + case CondGe: + return mergeQueryEntriesSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return less(v, rv); }, distinct, lhs); + case CondGt: + return mergeQueryEntriesSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return !less(rv, v); }, distinct, lhs); + case CondRange: + return mergeQueryEntriesSetNotSet( + lqe, rqe, + [&rv1 = rqe.Values()[0], &rv2 = rqe.Values()[1], &less](const Variant &v) { return less(v, rv1) || less(rv2, v); }, + distinct, lhs); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondAllSet: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesAllSetSet(lqe, rqe, distinct, lhs, collate); + case CondAllSet: + return mergeQueryEntriesAllSetAllSet(lqe, rqe, distinct, lhs, collate); + case CondLt: + return mergeQueryEntriesAllSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return !less(v, rv); }, distinct, lhs, collate); + case CondLe: + return mergeQueryEntriesAllSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return less(rv, v); }, distinct, lhs, collate); + case CondGe: + return mergeQueryEntriesAllSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return less(v, rv); }, distinct, lhs, collate); + case CondGt: + return mergeQueryEntriesAllSetNotSet( + lqe, rqe, [&rv = rqe.Values()[0], &less](const Variant &v) { return !less(rv, v); }, distinct, lhs, collate); + case CondRange: + return mergeQueryEntriesAllSetNotSet( + lqe, rqe, + [&rv1 = rqe.Values()[0], &rv2 = rqe.Values()[1], &less](const Variant &v) { return less(v, rv1) || less(rv2, v); }, + distinct, lhs, collate); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondLt: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return !less(v, lv); }, distinct, lhs); + case CondAllSet: + return mergeQueryEntriesAllSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return !less(v, lv); }, distinct, lhs, collate); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondLt: + case CondLe: + return mergeQueryEntriesLt(lqe, rqe, distinct, collate); + case CondGt: + case CondGe: + return mergeQueryEntriesLtGt(lqe, rqe, lhs, collate); + case CondRange: + return mergeQueryEntriesRangeLt(rqe, lqe, distinct, lhs, collate); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondLe: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return less(lv, v); }, distinct, lhs); + case CondAllSet: + return mergeQueryEntriesAllSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return less(lv, v); }, distinct, lhs, collate); + case CondLt: + case CondLe: + return mergeQueryEntriesLt(lqe, rqe, distinct, collate); + case CondGt: + return mergeQueryEntriesLtGt(lqe, rqe, lhs, collate); + case CondGe: + return mergeQueryEntriesLeGe(lqe, rqe, distinct, lhs, collate); + case CondRange: + return mergeQueryEntriesRangeLe(rqe, lqe, distinct, lhs, collate); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondGt: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return !less(lv, v); }, distinct, lhs); + case CondAllSet: + return mergeQueryEntriesAllSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return !less(lv, v); }, distinct, lhs, collate); + case CondGt: + case CondGe: + return mergeQueryEntriesGt(lqe, rqe, distinct, collate); + case CondLt: + case CondLe: + return mergeQueryEntriesLtGt(rqe, lqe, lhs, collate); + case CondRange: + return mergeQueryEntriesRangeGt(rqe, lqe, distinct, lhs, collate); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondGe: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return less(v, lv); }, distinct, lhs); + case CondAllSet: + return mergeQueryEntriesAllSetNotSet( + rqe, lqe, [&lv = lqe.Values()[0], &less](const Variant &v) { return less(v, lv); }, distinct, lhs, collate); + case CondGt: + case CondGe: + return mergeQueryEntriesGt(lqe, rqe, distinct, collate); + case CondLt: + return mergeQueryEntriesLtGt(rqe, lqe, lhs, collate); + case CondLe: + return mergeQueryEntriesLeGe(rqe, lqe, distinct, lhs, collate); + case CondRange: + return mergeQueryEntriesRangeGe(rqe, lqe, distinct, lhs, collate); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondRange: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + return mergeQueryEntriesSetNotSet( + rqe, lqe, + [&lv1 = lqe.Values()[0], &lv2 = lqe.Values()[1], &less](const Variant &v) { return less(v, lv1) || less(lv2, v); }, + distinct, lhs); + case CondAllSet: + return mergeQueryEntriesAllSetNotSet( + rqe, lqe, + [&lv1 = lqe.Values()[0], &lv2 = lqe.Values()[1], &less](const Variant &v) { return less(v, lv1) || less(lv2, v); }, + distinct, lhs, collate); + case CondLt: + return mergeQueryEntriesRangeLt(lqe, rqe, distinct, lhs, collate); + case CondLe: + return mergeQueryEntriesRangeLe(lqe, rqe, distinct, lhs, collate); + case CondGt: + return mergeQueryEntriesRangeGt(lqe, rqe, distinct, lhs, collate); + case CondGe: + return mergeQueryEntriesRangeGe(lqe, rqe, distinct, lhs, collate); + case CondRange: + return mergeQueryEntriesRange(lqe, rqe, distinct, lhs, collate); + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondDWithin: + case CondLike: + return MergeResult::NotMerged; + } + break; + case CondAny: + return mergeQueryEntriesAny(lqe, rqe, distinct); + case CondEmpty: + switch (rqe.Condition()) { + case CondEq: + case CondSet: + case CondAllSet: + case CondLt: + case CondLe: + case CondGe: + case CondGt: + case CondRange: + case CondDWithin: + case CondLike: + case CondAny: + SetValue(lhs, AlwaysFalse{}); + return MergeResult::Annihilated; + case CondEmpty: + lqe.Distinct(distinct); + return MergeResult::Merged; + } + break; + case CondDWithin: + switch (rqe.Condition()) { + case CondDWithin: + return mergeQueryEntriesDWithin(lqe, rqe, distinct, lhs); + case CondEq: + case CondSet: + case CondAllSet: + case CondLt: + case CondLe: + case CondGe: + case CondGt: + case CondRange: + case CondLike: + case CondAny: + case CondEmpty: + return MergeResult::NotMerged; + } + break; + case CondLike: + return MergeResult::NotMerged; + } + return MergeResult::NotMerged; } void QueryPreprocessor::AddDistinctEntries(const h_vector &aggregators) { diff --git a/cpp_src/core/nsselecter/querypreprocessor.h b/cpp_src/core/nsselecter/querypreprocessor.h index 7248bdd80..49a9ca884 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.h +++ b/cpp_src/core/nsselecter/querypreprocessor.h @@ -67,6 +67,9 @@ class QueryPreprocessor : private QueryEntries { static void SetQueryField(QueryField &, const NamespaceImpl &); private: + enum class NeedSwitch : bool { Yes = true, No = false }; + enum class MergeResult { NotMerged, Merged, Annihilated }; + enum class MergeOrdered : bool { Yes = true, No = false }; struct FoundIndexInfo { enum class ConditionType { Incompatible = 0, Compatible = 1 }; @@ -83,7 +86,34 @@ class QueryPreprocessor : private QueryEntries { bool forcedStage() const noexcept { return evaluationsCount_ == (desc_ ? 1 : 0); } size_t lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, uint16_t srcEnd); size_t substituteCompositeIndexes(size_t from, size_t to); - bool mergeQueryEntries(size_t lhs, size_t rhs); + template + MergeResult mergeQueryEntries(size_t lhs, size_t rhs, const CollateOpts &); + MergeResult mergeQueryEntriesSetSet(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesAllSetSet(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, const CollateOpts &); + MergeResult mergeQueryEntriesAllSetAllSet(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesAny(QueryEntry &lqe, QueryEntry &rqe, bool distinct); + template + MergeResult mergeQueryEntriesSetNotSet(QueryEntry &lqe, QueryEntry &rqe, F filter, bool distinct, size_t position); + template + MergeResult mergeQueryEntriesAllSetNotSet(QueryEntry &lqe, QueryEntry &rqe, F filter, bool distinct, size_t position, + const CollateOpts &); + MergeResult mergeQueryEntriesDWithin(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position); + MergeResult mergeQueryEntriesLt(QueryEntry &lqe, QueryEntry &rqe, bool distinct, const CollateOpts &); + MergeResult mergeQueryEntriesGt(QueryEntry &lqe, QueryEntry &rqe, bool distinct, const CollateOpts &); + MergeResult mergeQueryEntriesLtGt(QueryEntry &lqe, QueryEntry &rqe, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesLeGe(QueryEntry &lqe, QueryEntry &rqe, bool distinct, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesRangeLt(QueryEntry &range, QueryEntry &ge, bool distinct, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesRangeLe(QueryEntry &range, QueryEntry &ge, bool distinct, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesRangeGt(QueryEntry &range, QueryEntry &ge, bool distinct, size_t position, const CollateOpts &); + template + MergeResult mergeQueryEntriesRangeGe(QueryEntry &range, QueryEntry &ge, bool distinct, size_t position, const CollateOpts &); + MergeResult mergeQueryEntriesRange(QueryEntry &range, QueryEntry &ge, bool distinct, size_t position, const CollateOpts &); const std::vector *getCompositeIndex(int field) const; void convertWhereValues(QueryEntries::iterator begin, QueryEntries::iterator end) const; void convertWhereValues(QueryEntry &) const; @@ -102,9 +132,14 @@ class QueryPreprocessor : private QueryEntries { [[nodiscard]] std::pair queryValuesFromOnCondition(CondType condition, const QueryJoinEntry &, const JoinedSelector &, const CollateOpts &); void checkStrictMode(const QueryField &) const; - bool removeBrackets(); - size_t removeBrackets(size_t begin, size_t end); - bool canRemoveBracket(size_t i) const; + [[nodiscard]] bool removeBrackets(); + [[nodiscard]] size_t removeBrackets(size_t begin, size_t end); + [[nodiscard]] bool canRemoveBracket(size_t i) const; + [[nodiscard]] bool removeAlwaysFalse(); + [[nodiscard]] std::pair removeAlwaysFalse(size_t begin, size_t end); + [[nodiscard]] bool removeAlwaysTrue(); + [[nodiscard]] std::pair removeAlwaysTrue(size_t begin, size_t end); + [[nodiscard]] bool containsJoin(size_t) noexcept; template void briefDump(size_t from, size_t to, const std::vector &joinedSelectors, WrSerializer &ser) const; diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.cc b/cpp_src/core/nsselecter/selectiteratorcontainer.cc index dfa14114e..28b377053 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.cc +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.cc @@ -155,6 +155,18 @@ bool SelectIteratorContainer::isIdset(const_iterator it, const_iterator end) { (++it == end || it->operation != OpOr); } +bool SelectIteratorContainer::isAlwaysFalse(const_iterator it, const_iterator end) { + return ((it->operation == OpAnd && it->Is()) || (it->operation == OpNot && it->Is())) && + (++it == end || it->operation != OpOr); +} + +bool SelectIteratorContainer::HasAlwaysFalse() const { + for (const_iterator it = cbegin(), end = cend(); it != end; ++it) { + if (isAlwaysFalse(it, end)) return true; + } + return false; +} + bool SelectIteratorContainer::HasIdsets() const { for (const_iterator it = cbegin(), end = cend(); it != end; ++it) { if (isIdset(it, end)) return true; diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.h b/cpp_src/core/nsselecter/selectiteratorcontainer.h index 4f78afc6f..88441a5ac 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.h +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.h @@ -39,6 +39,7 @@ class SelectIteratorContainer : public ExpressionTree static IdType getNextItemId(const_iterator begin, const_iterator end, IdType from); static bool isIdset(const_iterator it, const_iterator end); + static bool isAlwaysFalse(const_iterator it, const_iterator end); static bool markBracketsHavingJoins(iterator begin, iterator end) noexcept; bool haveJoins(size_t i) const noexcept; diff --git a/cpp_src/core/query/queryentry.cc b/cpp_src/core/query/queryentry.cc index 74b62dfab..ca228bba9 100644 --- a/cpp_src/core/query/queryentry.cc +++ b/cpp_src/core/query/queryentry.cc @@ -160,17 +160,16 @@ template void VerifyQueryEntryValues(C std::string QueryEntry::Dump() const { WrSerializer ser; if (Distinct()) { - ser << "Distinct index: " << FieldName(); - } else { - ser << FieldName() << ' ' << condition_ << ' '; - const bool severalValues = (Values().size() > 1); - if (severalValues) ser << '('; - for (auto &v : Values()) { - if (&v != &*Values().begin()) ser << ','; - ser << '\'' << v.As() << '\''; - } - if (severalValues) ser << ')'; + ser << "Distinct index: "; + } + ser << FieldName() << ' ' << condition_ << ' '; + const bool severalValues = (Values().size() > 1); + if (severalValues) ser << '('; + for (auto &v : Values()) { + if (&v != &*Values().begin()) ser << ','; + ser << '\'' << v.As() << '\''; } + if (severalValues) ser << ')'; return std::string{ser.Slice()}; } @@ -551,7 +550,7 @@ void QueryEntries::dump(size_t level, const_iterator begin, const_iterator end, [&ser, subQueries](const SubQueryFieldEntry &sqe) { ser << sqe.Dump(subQueries); }, [&](const QueryEntriesBracket &b) { ser << "(\n"; - dump(level + 1, it.cbegin(), it.cend(), joinedSelectors, ser); + dump(level + 1, it.cbegin(), it.cend(), joinedSelectors, subQueries, ser); dumpEqualPositions(level + 1, ser, b.equalPositions); for (size_t i = 0; i < level; ++i) { ser << " "; @@ -561,9 +560,13 @@ void QueryEntries::dump(size_t level, const_iterator begin, const_iterator end, [&ser](const QueryEntry &qe) { ser << qe.Dump() << '\n'; }, [&joinedSelectors, &ser](const JoinQueryEntry &jqe) { ser << jqe.Dump(joinedSelectors) << '\n'; }, [&ser](const BetweenFieldsQueryEntry &qe) { ser << qe.Dump() << '\n'; }, - [&ser](const AlwaysFalse &) { ser << "AlwaysFalse" << 'n'; }, - [&ser](const AlwaysTrue &) { ser << "AlwaysTrue" << 'n'; }); + [&ser](const AlwaysFalse &) { ser << "AlwaysFalse\n"; }, + [&ser](const AlwaysTrue &) { ser << "AlwaysTrue\n"; }); } } +template void QueryEntries::dump(size_t, const_iterator, const_iterator, const std::vector &, const std::vector &, + WrSerializer &); +template void QueryEntries::dump(size_t, const_iterator, const_iterator, const std::vector &, + const std::vector &, WrSerializer &); } // namespace reindexer diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index 84ef66a88..1e436f4a9 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -112,6 +112,9 @@ class QueryEntry : private QueryField { [[nodiscard]] auto UpdatableValues(IgnoreEmptyValues) & noexcept { return VerifyingUpdater{*this}; } + [[nodiscard]] auto UpdatableValues() & noexcept { + return VerifyingUpdater{*this}; + } [[nodiscard]] bool Distinct() const noexcept { return distinct_; } void Distinct(bool d) noexcept { distinct_ = d; } using QueryField::IndexNo; @@ -159,6 +162,7 @@ class QueryEntry : private QueryField { private: void verifyIgnoringEmptyValues() const { VerifyQueryEntryValues(condition_, values_); } + void verifyNotIgnoringEmptyValues() const { VerifyQueryEntryValues(condition_, values_); } VariantArray values_; CondType condition_; diff --git a/cpp_src/estl/forward_like.h b/cpp_src/estl/forward_like.h new file mode 100644 index 000000000..66292d0d6 --- /dev/null +++ b/cpp_src/estl/forward_like.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace reindexer { + +template +[[nodiscard]] constexpr auto&& forward_like(U&& x) noexcept { + constexpr bool is_adding_const = std::is_const_v>; + if constexpr (std::is_lvalue_reference_v) { + if constexpr (is_adding_const) { + return std::as_const(x); + } else { + return static_cast(x); + } + } else { + if constexpr (is_adding_const) { + return std::move(std::as_const(x)); + } else { + return std::move(x); + } + } +} + +} // namespace reindexer diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index a1d79cc76..c30da3e1a 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -651,11 +651,18 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { return item; } - void RandVariantArray(size_t size, size_t max, VariantArray& arr) { + VariantArray RandVariantArray(size_t size, size_t min, size_t range) { + VariantArray result; + RandVariantArray(size, min, min + range, result); + return result; + } + + void RandVariantArray(size_t size, size_t min, size_t max, VariantArray& arr) { + assert(min < max); arr.clear(); arr.reserve(size); for (size_t i = 0; i < size; ++i) { - arr.emplace_back(int(rand() % max)); + arr.emplace_back(int(rand() % (max - min) + min)); } } @@ -740,20 +747,47 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { } void CheckConditionsMergingQueries() { + const auto randCond = [conds = {CondEq, CondLt, CondLe, CondGt, CondGe}]() noexcept { + return *(conds.begin() + rand() % std::size(conds)); + }; // Check merging of conditions by the same index with large sets of values - VariantArray arr1, arr2, emptyArr; + int64_t tmp; for (size_t i = 0; i < 3; ++i) { - RandVariantArray(500, 1000, arr1); - RandVariantArray(100, 1000, arr2); - arr2.insert(arr2.end(), arr1.begin(), arr1.begin() + 100); + struct { + CondType cond; + VariantArray values; + } testData[]{ + {CondSet, RandVariantArray(500, 0, 1000)}, + {CondSet, RandVariantArray(100, 0, 1000)}, + {CondSet, {}}, + {CondSet, RandVariantArray(rand() % 4, rand() % 1000, rand() % 100 + 1)}, + {CondRange, (tmp = rand() % 1000, VariantArray::Create(tmp, rand() % 1000 + tmp))}, + {CondRange, (tmp = rand() % 1000, VariantArray::Create(tmp, (rand() % 2) * 100 + tmp))}, + }; + testData[1].values.insert(testData[1].values.end(), testData[0].values.begin(), testData[0].values.begin() + 100); - ExecuteAndVerifyWithSql( - Query(default_namespace).Where(kFieldNameNumeric, CondSet, arr1).Where(kFieldNameNumeric, CondSet, arr2)); + ExecuteAndVerifyWithSql(Query(default_namespace) + .Where(kFieldNameNumeric, testData[0].cond, testData[0].values) + .Where(kFieldNameNumeric, testData[1].cond, testData[1].values)); ExecuteAndVerifyWithSql(Query(default_namespace) - .Where(kFieldNameNumeric, CondSet, arr1) - .Where(kFieldNameNumeric, CondSet, emptyArr) - .Where(kFieldNameNumeric, CondSet, arr2)); + .Where(kFieldNameNumeric, testData[0].cond, testData[0].values) + .Where(kFieldNameNumeric, testData[2].cond, testData[2].values) + .Where(kFieldNameNumeric, testData[1].cond, testData[1].values)); + for (size_t j = 0; j < 10; ++j) { + Query q{default_namespace}; + for (size_t l = 0, n = rand() % 10 + 2; l < n; ++l) { + const size_t testCase = rand() % (std::size(testData) * 2 + 1); + if (testCase < std::size(testData)) { + q.Where(kFieldNameNumeric, testData[testCase].cond, testData[testCase].values); + } else if (testCase == std::size(testData)) { + q.Where(kFieldNameNumeric, rand() % 2 ? CondAny : CondEmpty, VariantArray{}); + } else { + q.Where(kFieldNameNumeric, randCond(), VariantArray::Create(rand() % 1000)); + } + } + ExecuteAndVerifyWithSql(q); + } } } diff --git a/cpp_src/gtests/tests/unit/ft/ft_generic.cc b/cpp_src/gtests/tests/unit/ft/ft_generic.cc index 63d9e9c86..0d3deba95 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_generic.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_generic.cc @@ -1366,8 +1366,8 @@ TEST_P(FTGenericApi, ExplainWithFtPreselect) { auto selectors = root["selectors"]; ASSERT_TRUE(selectors.IsSequence()) << qr.explainResults; ASSERT_EQ(selectors.size(), 2) << qr.explainResults; - ASSERT_EQ(selectors[0]["field"].as(), "(-scan and (id and inner_join ns_for_joins) or id)") << qr.explainResults; - ASSERT_EQ(selectors[1]["field"].as(), "ft3") << qr.explainResults; + EXPECT_EQ(selectors[0]["field"].as(), "(-scan and (id and inner_join ns_for_joins) or id)") << qr.explainResults; + EXPECT_EQ(selectors[1]["field"].as(), "ft3") << qr.explainResults; } } diff --git a/cpp_src/gtests/tests/unit/selector_plan_test.cc b/cpp_src/gtests/tests/unit/selector_plan_test.cc index 2f96be009..bad365d53 100644 --- a/cpp_src/gtests/tests/unit/selector_plan_test.cc +++ b/cpp_src/gtests/tests/unit/selector_plan_test.cc @@ -90,17 +90,19 @@ TEST_F(SelectorPlanTest, SortByBtreeIndex) { ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "sort_by_uncommitted_index", {false})); ASSERT_NO_FATAL_FAILURE(AssertJsonFieldAbsent(explain, "items")); const auto cost = GetJsonFieldValues(explain, "cost"); - ASSERT_EQ(2, cost.size()); - ASSERT_LE(cost[0], cost[1]); - if (searchByBtreeField) { - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "sort_index", {searchField})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "index"})); - } else { - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "sort_index", {"-"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 1})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "scan"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "type", {"SingleIdset", "OnlyComparator"})); + if (additionalSearchField != searchField) { + ASSERT_EQ(2, cost.size()); + ASSERT_LE(cost[0], cost[1]); + if (searchByBtreeField) { + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "sort_index", {searchField})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 0})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "index"})); + } else { + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "sort_index", {"-"})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 1})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "scan"})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "type", {"SingleIdset", "OnlyComparator"})); + } } } } @@ -170,19 +172,21 @@ TEST_F(SelectorPlanTest, SortByBtreeIndex) { ASSERT_NO_FATAL_FAILURE(AssertJsonFieldAbsent(explain, "items")); ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "sort_index", {sortByBtreeField ? sortField : "-"})); const auto cost = GetJsonFieldValues(explain, "cost"); - ASSERT_EQ(2, cost.size()); - ASSERT_LE(cost[0], cost[1]); - if (searchByBtreeField) { - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "index"})); - } else { - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 1})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "scan"})); - if (sortByBtreeField) { - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo( - explain, "type", {desc ? "RevSingleIdset" : "SingleIdset", "OnlyComparator"})); + if (additionalSearchField != searchField) { + ASSERT_EQ(2, cost.size()); + ASSERT_LE(cost[0], cost[1]); + if (searchByBtreeField) { + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 0})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "index"})); } else { - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "type", {"SingleIdset", "OnlyComparator"})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "comparators", {0, 1})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "method", {"index", "scan"})); + if (sortByBtreeField) { + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo( + explain, "type", {desc ? "RevSingleIdset" : "SingleIdset", "OnlyComparator"})); + } else { + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(explain, "type", {"SingleIdset", "OnlyComparator"})); + } } } } @@ -229,6 +233,7 @@ TEST_F(SelectorPlanTest, SortByUnbuiltBtreeIndex) { } for (const char* additionalSearchField : {kFieldId, kFieldTree1, kFieldTree2, kFieldHash}) { + if (additionalSearchField == searchField) continue; for (const Query& query : {Query(unbuiltBtreeNs).Explain().Where(additionalSearchField, CondEq, RandInt()).Where(searchField, cond, RandInt()), Query(unbuiltBtreeNs) @@ -335,6 +340,7 @@ TEST_F(SelectorPlanTest, SortByUnbuiltBtreeIndex) { } for (const char* additionalSearchField : {kFieldId, kFieldTree1, kFieldTree2, kFieldHash}) { + if (additionalSearchField == searchField) continue; for (const Query& query : {Query(unbuiltBtreeNs) .Explain() .Where(additionalSearchField, CondEq, RandInt()) @@ -414,59 +420,33 @@ TEST_F(SelectorPlanTest, ConditionsMergeIntoEmptyCondition) { Upsert(nsName, item); } - { - // Query without intersection in CondEq/CondSet values - const auto q = Query(nsName) - .Where("id", CondEq, 31) - .Where("id", CondSet, {32, 33, 34}) - .Where("id", CondEq, 310) - .Where("id", CondSet, {35, 36, 37}) - .Where("value", CondAny, VariantArray{}) - .Explain(); - QueryResults qr; - err = rt.reindexer->Select(q, qr); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(qr.Count(), 0); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "field", {"id", "value"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "keys", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "matched", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "method", {"index", "scan"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "type", {"OnlyComparator", "OnlyComparator"})); - } - { - // Query with empty set - const auto q = Query(nsName) - .Where("id", CondEq, 39) - .Where("id", CondSet, {32, 39, 34}) - .Where("id", CondSet, VariantArray{}) - .Where("value", CondAny, VariantArray{}) - .Explain(); - QueryResults qr; - err = rt.reindexer->Select(q, qr); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(qr.Count(), 0); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "field", {"id", "value"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "keys", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "matched", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "method", {"index", "scan"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "type", {"OnlyComparator", "OnlyComparator"})); - } - { - // Query with multiple empty sets - const auto q = Query(nsName) - .Where("id", CondEq, 45) - .Where("id", CondSet, VariantArray{}) - .Where("id", CondSet, VariantArray{}) - .Where("value", CondAny, VariantArray{}) - .Explain(); + for (const Query& q : {Query(nsName) // Query without intersection in CondEq/CondSet values + .Where("id", CondEq, 31) + .Where("id", CondSet, {32, 33, 34}) + .Where("id", CondEq, 310) + .Where("id", CondSet, {35, 36, 37}) + .Where("value", CondAny, VariantArray{}) + .Explain(), + Query(nsName) // Query with empty set + .Where("id", CondEq, 39) + .Where("id", CondSet, {32, 39, 34}) + .Where("id", CondSet, VariantArray{}) + .Where("value", CondAny, VariantArray{}) + .Explain(), + Query(nsName) // Query with multiple empty sets + .Where("id", CondEq, 45) + .Where("id", CondSet, VariantArray{}) + .Where("id", CondSet, VariantArray{}) + .Where("value", CondAny, VariantArray{}) + .Explain()}) { QueryResults qr; err = rt.reindexer->Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.Count(), 0); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "field", {"id", "value"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "keys", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "matched", {0, 0})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "method", {"index", "scan"})); - ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "type", {"OnlyComparator", "OnlyComparator"})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "field", {"always_false"})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "keys", {1})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "matched", {0})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "method", {"index"})); + ASSERT_NO_FATAL_FAILURE(AssertJsonFieldEqualTo(qr.GetExplainResults(), "type", {"SingleRange", "Skipped"})); } } diff --git a/fulltext.md b/fulltext.md index 1d13e8b53..916788798 100644 --- a/fulltext.md +++ b/fulltext.md @@ -388,7 +388,7 @@ Several parameters of full text search engine can be configured from application The list item can be either a string or a structure containing a string (the stopword) and a bool attribute (`is_morpheme`) indicating whether the stopword can be part of a word that can be shown in query-results. If the stopword is set as a string, then the `is_morpheme` attribute is `false` by default and following entries are equivalent: ```json -"stop_words":[ +"StopWords":[ { "word": "some_word", "is_morpheme": false @@ -398,7 +398,7 @@ If the stopword is set as a string, then the `is_morpheme` attribute is `false` ``` , ```json -"stop_words":[ +"StopWords":[ "some_word", ///... ] @@ -407,7 +407,7 @@ If the stopword is set as a string, then the `is_morpheme` attribute is `false` #### Example: If the list of stopwords looks like this: ```json -"stop_words":[ +"StopWords":[ { "word": "under", "is_morpheme": true diff --git a/test/queries_test.go b/test/queries_test.go index 191c84146..643d5c36c 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -2205,12 +2205,6 @@ func TestQueryExplain(t *testing.T) { Comparators: 0, Matched: 5, }, - { - Description: "always true", - Keys: 0, - Comparators: 0, - Matched: 0, - }, }, "") checkExplainSubqueries(t, explainRes.SubQueriesExplains, []expectedExplainSubQuery{ { @@ -2288,11 +2282,11 @@ func TestQueryExplain(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "-scan", - Method: "scan", - Keys: 0, + Field: "always_false", + Method: "index", + Keys: 1, Comparators: 0, - Matched: 1, + Matched: 0, }, { Description: "always false", @@ -2300,14 +2294,6 @@ func TestQueryExplain(t *testing.T) { Comparators: 0, Matched: 0, }, - { - Field: "id", - FieldType: "indexed", - Method: "scan", - Keys: 0, - Comparators: 1, - Matched: 0, - }, }, "") checkExplainSubqueries(t, explainRes.SubQueriesExplains, []expectedExplainSubQuery{ {