Skip to content

Commit

Permalink
Merge branch 'develop' into 1661_482_merge_queryEntries
Browse files Browse the repository at this point in the history
  • Loading branch information
reindexer-bot committed Feb 2, 2024
1 parent 36d596b commit fdc8799
Show file tree
Hide file tree
Showing 16 changed files with 1,253 additions and 200 deletions.
195 changes: 195 additions & 0 deletions cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
#!/bin/bash
# Task: https://github.com/restream/reindexer/-/issues/1188
set -e

function KillAndRemoveServer {
local pid=$1
kill $pid
wait $pid
yum remove -y 'reindexer*' > /dev/null
}

function WaitForDB {
# wait until DB is loaded
set +e # disable "exit on error" so the script won't stop when DB's not loaded yet
is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list');
while [[ $is_connected != "test" ]]
do
sleep 2
is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list');
done
set -e
}

function CompareNamespacesLists {
local ns_list_actual=$1
local ns_list_expected=$2
local pid=$3

diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order
if [ "$diff" == "" ]; then
echo "## PASS: namespaces list not changed"
else
echo "##### FAIL: namespaces list was changed"
echo "expected: $ns_list_expected"
echo "actual: $ns_list_actual"
KillAndRemoveServer $pid;
exit 1
fi
}

function CompareMemstats {
local actual=$1
local expected=$2
local pid=$3
diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order
if [ "$diff" == "" ]; then
echo "## PASS: memstats not changed"
else
echo "##### FAIL: memstats was changed"
echo "expected: $expected"
echo "actual: $actual"
KillAndRemoveServer $pid;
exit 1
fi
}


RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)"
VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..')
VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version

echo "## choose latest release rpm file"
if [ $VERSION == 3 ]; then
LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 3)
namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg'
elif [ $VERSION == 4 ]; then
LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 4)
# replicationstats ns added for v4
namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg'
else
echo "Unknown version"
exit 1
fi

echo "## downloading latest release rpm file: $LATEST_RELEASE"
curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE;
echo "## downloading example DB"
curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip;
unzip -o big.zip # unzips into mydb_big.rxdump;

ADDRESS="cproto://127.0.0.1:6534/"
DB_NAME="test"

memstats_expected=$'[
{"replication":{"data_hash":24651210926,"data_count":3}},
{"replication":{"data_hash":6252344969,"data_count":1}},
{"replication":{"data_hash":37734732881,"data_count":28}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":1024095024522,"data_count":1145}},
{"replication":{"data_hash":8373644068,"data_count":1315}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":7404222244,"data_count":97}},
{"replication":{"data_hash":94132837196,"data_count":4}},
{"replication":{"data_hash":1896088071,"data_count":2}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":-672103903,"data_count":33538}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":6833710705,"data_count":1}},
{"replication":{"data_hash":5858155773472,"data_count":4500}},
{"replication":{"data_hash":-473221280268823592,"data_count":65448}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":8288213744,"data_count":3}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":0,"data_count":0}},
{"replication":{"data_hash":354171024786967,"data_count":3941}},
{"replication":{"data_hash":-6520334670,"data_count":35886}},
{"replication":{"data_hash":112772074632,"data_count":281}},
{"replication":{"data_hash":-12679568198538,"data_count":1623116}}
]
Returned 27 rows'

echo "##### Forward compatibility test #####"

DB_PATH=$(pwd)"/rx_db"

echo "Database: "$DB_PATH

echo "## installing latest release: $LATEST_RELEASE"
yum install -y $LATEST_RELEASE > /dev/null;
# run RX server with disabled logging
reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb;
sleep 1;

namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_1;
CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid;

memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid;

KillAndRemoveServer $server_pid;

echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM"
yum install -y build/*.rpm > /dev/null;
reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

WaitForDB

namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_2;
CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid;

memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid;

KillAndRemoveServer $server_pid;
rm -rf $DB_PATH;
sleep 1;

echo "##### Backward compatibility test #####"

echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM"
yum install -y build/*.rpm > /dev/null;
reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb;
sleep 1;

namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_3;
CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid;

memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid;

KillAndRemoveServer $server_pid;

echo "## installing latest release: $LATEST_RELEASE"
yum install -y $LATEST_RELEASE > /dev/null;
reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH &
server_pid=$!
sleep 2;

WaitForDB

namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list');
echo $namespaces_4;
CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid;

memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats');
CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid;

KillAndRemoveServer $server_pid;
rm -rf $DB_PATH;
16 changes: 16 additions & 0 deletions cpp_src/core/keyvalue/fast_hash_set_variant.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#pragma once

#include "estl/fast_hash_set.h"
#include "variant.h"

namespace reindexer {

class fast_hash_set_variant : public fast_hash_set<Variant, std::hash<Variant>, Variant::EqualTo, Variant::Less> {
using Base = fast_hash_set<Variant, std::hash<Variant>, Variant::EqualTo, Variant::Less>;

public:
fast_hash_set_variant(const CollateOpts& collate)
: Base{16, std::hash<Variant>{}, Variant::EqualTo{collate}, typename Base::allocator_type{}, Variant::Less{collate}} {}
};

} // namespace reindexer
25 changes: 22 additions & 3 deletions cpp_src/core/keyvalue/variant.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class Variant {
template <typename... Ts>
Variant(const std::tuple<Ts...> &);

Variant &operator=(Variant &&other) &noexcept {
Variant &operator=(Variant &&other) & noexcept {
if (this == &other) return *this;
if (isUuid()) {
if (other.isUuid()) {
Expand Down Expand Up @@ -119,6 +119,7 @@ class Variant {
bool operator<(const Variant &other) const { return Compare(other) < 0; }
bool operator>(const Variant &other) const { return Compare(other) > 0; }
bool operator>=(const Variant &other) const { return Compare(other) >= 0; }
bool operator<=(const Variant &other) const { return Compare(other) <= 0; }

int Compare(const Variant &other, const CollateOpts &collateOpts = CollateOpts()) const;
template <WithString>
Expand Down Expand Up @@ -147,6 +148,24 @@ class Variant {
template <typename T>
void Dump(T &os, CheckIsStringPrintable checkPrintableString = CheckIsStringPrintable::Yes) const;

class Less {
public:
Less(const CollateOpts &collate) noexcept : collate_{&collate} {}
[[nodiscard]] bool operator()(const Variant &lhs, const Variant &rhs) const { return lhs.Compare(rhs, *collate_) < 0; }

private:
const CollateOpts *collate_;
};

class EqualTo {
public:
EqualTo(const CollateOpts &collate) noexcept : collate_{&collate} {}
[[nodiscard]] bool operator()(const Variant &lhs, const Variant &rhs) const { return lhs.Compare(rhs, *collate_) == 0; }

private:
const CollateOpts *collate_;
};

private:
bool isUuid() const noexcept { return uuid_.isUuid != 0; }
void convertToComposite(const PayloadType &, const FieldsSet &);
Expand Down Expand Up @@ -223,11 +242,11 @@ class VariantArray : public h_vector<Variant, 2> {
VariantArray() noexcept = default;
explicit VariantArray(Point) noexcept;
explicit operator Point() const;
VariantArray &MarkArray(bool v = true) &noexcept {
VariantArray &MarkArray(bool v = true) & noexcept {
isArrayValue = v;
return *this;
}
VariantArray &&MarkArray(bool v = true) &&noexcept {
VariantArray &&MarkArray(bool v = true) && noexcept {
isArrayValue = v;
return std::move(*this);
}
Expand Down
8 changes: 6 additions & 2 deletions cpp_src/core/nsselecter/nsselecter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,14 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
fnc_ = ctx.functions->AddNamespace(ctx.query, *ns_, ctx.nsid, isFt);
}

qPreproc.ConvertWhereValues();
if (!ctx.skipIndexesLookup) {
qPreproc.Reduce(isFt);
}
if (isFt) {
qPreproc.CheckUniqueFtQuery();
qPreproc.ExcludeFtQuery(rdxCtx);
}
qPreproc.ConvertWhereValues();

explain.AddPrepareTime();

Expand Down Expand Up @@ -252,7 +252,11 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
if (it.comparators_.size()) hasComparators = true;
});

if (!qres.HasIdsets()) {
if (qres.HasAlwaysFalse()) {
SelectKeyResult zeroScan;
zeroScan.emplace_back(0, 0);
qres.AppendFront(OpAnd, SelectIterator{std::move(zeroScan), false, "always_false", IteratorFieldKind::None, true});
} else if (!qres.HasIdsets()) {
SelectKeyResult scan;
if (ctx.sortingContext.isOptimizationEnabled()) {
auto it = ns_->indexes_[ctx.sortingContext.uncommitedIndex]->CreateIterator();
Expand Down
Loading

0 comments on commit fdc8799

Please sign in to comment.