Update to version v3.22.1

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Build Reindexer
         run: |
           mkdir build && cd build
@@ -27,7 +27,7 @@ jobs:
       fail-fast: false
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Build Reindexer
         run: |
           mkdir build && cd build
@@ -57,7 +57,7 @@ jobs:
       CXX: ${{matrix.cxx}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Prepare Environment
         run: |
           ./.github/workflows/install_gtest.sh
@@ -94,7 +94,7 @@ jobs:
           rm -rf ./build/_CPack_Packages ./build/cpp_src/server/reindexer_server_resources
           tar -cvf artifacts.tar build/ bindings/builtin/builtin_posix.go bindings/builtinserver/builtinserver_posix.go dependencies.sh
       - name: Archive Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{matrix.os}}${{matrix.sanitizer}}
           path: artifacts.tar
@@ -133,10 +133,10 @@ jobs:
     steps:
       - name: Checkout repository
         if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Download ${{matrix.os}}${{matrix.sanitizer}} Artifacts
         if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.os}}${{matrix.sanitizer}}
       - name: 'Untar Artifacts'
@@ -183,7 +183,7 @@ jobs:
       OS: ${{matrix.os}}
     steps:
       - name: Download ${{matrix.os}} Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.os}}
       - name: 'Untar Artifacts'
@@ -217,7 +217,7 @@ jobs:
             cp ./usr/local/etc/reindexer.conf.pkg /usr/local/etc/reindexer.conf.pkg
           fi
       - name: Clone PyReindexer
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: restream/reindexer-py
       - name: Install PyReindexer

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.22.0"
+const ReindexerVersion = "v3.22.1"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,21 +1,34 @@
+# Version 3.22.1 (12.02.2024)
+## Core
+- [fix] Fixed `COUNT()` for fulltext queries, when `preselect_before_ft` is enabled in the index config
+- [fix] Fixed `LIMIT`/`OFFSET` behavior for `MERGE`-queries, when `preselect_before_ft` is enabled in the index config
+- [fix] Disabled in-memory WAL for the system namespaces
+
+## Replication
+- [fix] Improved transactions replication via WAL queries: from now transactions will not be splitted into separate insert/updates on the follower
+
+## Face
+- [fix] Fixed the Check request 
+- [fix] Added bottom padding to the Performance Updates chart
+
 # Version 3.22.0 (02.02.2024)
 ## Core
 - [fea] Added `explain` results for the [subqueries](readme.md#subqueries-nested-queries)
 - [fea] Added support for limit/offset in `Delete` and `Update` queries
 - [fea] Optimized ordered indexes' cache logic to achive more cache hits and more compact cache size
 - [fea] Added support for `COUNT_CACHED(*)`/`CachedTotal()` aggregation in the queries with INNER JOINS. Now it's possible to cache total count results for such queries
-- [fix] Fixed SQL parsing for combintaions of the [subqueries](readme.md#subqueries-nested-queries) and other conditions in the main query
-- [fix] Fixed [select functions](fulltext.md#using-select-functions) with '.' delimiter. Previously those functios were actually expected '=' as a delimiter
+- [fix] Fixed SQL parsing for combinations of the [subqueries](readme.md#subqueries-nested-queries) and other conditions in the main query
+- [fix] Fixed [select functions](fulltext.md#using-select-functions) with '.' delimiter. Previously those functions actually expected '=' as a delimiter
 
 ## Fulltext
 - [fea] Reworked logic for the stop-words. [More details](fulltext.md#stopwords-details)
 - [fea] Added config for the base ranking algorithm. Check `bm25_config` field in the [fulltext settings](fulltext.md#base-config-parameters)
 
 ## Replication
-- [fea] Fixed sync logic to allow runtime server ID changing
+- [fea] Fixed sync logic to allow modification of the server's ID at runtime
 
 ## Reindexer server
-- [fix] Fixed SQL suggests for subqueries and some kinds of the JOIN-queries
+- [fix] Fixed SQL suggests for subqueries and certain types of JOIN-queries
 
 ## Docker
 - [fea] Base docker image was updated to alpine 3.19
@@ -26,7 +39,7 @@
 
 ## Face
 - [fea] Added the subqueries field to the explain mode
-- [fea] Upgraded the Webpack to 5.х
+- [fea] Upgraded Webpack to 5.х
 - [fea] Added the default values to the NS config during the mode changing
 - [fix] Fixed the message about the outdated browser version after Chrome upgraded to v120.
 - [fix] Fixed the settings panel layout on the Performance page, which was overlapped by the message about the outdated browser version

cpp_src/CMakeLists.txt

@@ -43,7 +43,7 @@ else()
   option (LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "3.22.0")
+set (REINDEXER_VERSION_DEFAULT "3.22.1")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -449,7 +449,7 @@ if (WITH_CPPTRACE)
   ExternalProject_Add(
     cpptrace_lib
     GIT_REPOSITORY "https://github.com/jeremy-rifkin/cpptrace.git"
-    GIT_TAG "main"
+    GIT_TAG "v0.3.1"
     CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR}	
       -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}
       -DCPPTRACE_STATIC=On

cpp_src/core/index/indexordered.cc

@@ -136,7 +136,7 @@ SelectKeyResults IndexOrdered<T>::SelectKey(const VariantArray &keys, CondType c
 				idsCount = 0;
 				res.reserve(count);
 				for (auto it = ctx.startIt; it != ctx.endIt; ++it) {
-					assertrx(it != ctx.i_map->end());  // FIXME: assertrx_dbg
+					assertrx_dbg(it != ctx.i_map->end());
 					idsCount += it->second.Unsorted().Size();
 					res.emplace_back(it->second, ctx.sortId);
 				}

cpp_src/core/namespace/namespaceimpl.cc

@@ -115,7 +115,7 @@ NamespaceImpl::NamespaceImpl(const std::string& name, UpdatesObservers& observer
 	  queryCountCache_(
 		  std::make_unique<QueryCountCache>(config_.cacheConfig.queryCountCacheSize, config_.cacheConfig.queryCountHitsToCache)),
 	  joinCache_(std::make_unique<JoinCache>(config_.cacheConfig.joinCacheSize, config_.cacheConfig.joinHitsToCache)),
-	  wal_(config_.walSize),
+	  wal_(getWalSize(config_)),
 	  observers_(&observers),
 	  lastSelectTime_{0},
 	  cancelCommitCnt_{0},
@@ -281,7 +281,7 @@ void NamespaceImpl::OnConfigUpdated(DBConfigProvider& configProvider, const RdxC
 		updateSortedIdxCount();
 	}
 
-	if (wal_.Resize(config_.walSize)) {
+	if (wal_.Resize(getWalSize(config_))) {
 		logPrintf(LogInfo, "[%s] WAL has been resized lsn #%s, max size %ld", name_, repl_.lastLsn, wal_.Capacity());
 	}
 
@@ -2641,7 +2641,7 @@ void NamespaceImpl::LoadFromStorage(unsigned threadsCount, const RdxContext& ctx
 }
 
 void NamespaceImpl::initWAL(int64_t minLSN, int64_t maxLSN) {
-	wal_.Init(config_.walSize, minLSN, maxLSN, storage_);
+	wal_.Init(getWalSize(config_), minLSN, maxLSN, storage_);
 	// Fill existing records
 	for (IdType rowId = 0; rowId < IdType(items_.size()); rowId++) {
 		if (!items_[rowId].IsFree()) {

cpp_src/core/namespace/namespaceimpl.h

@@ -472,6 +472,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base {	 // NOLINT(*performance.
 			storage_.TryForceFlush();
 		}
 	}
+	size_t getWalSize(const NamespaceConfigData &cfg) const noexcept { return isSystem() ? int64_t(1) : std::max(cfg.walSize, int64_t(1)); }
 
 	PerfStatCounterMT updatePerfCounter_, selectPerfCounter_;
 	std::atomic<bool> enablePerfCounters_;

cpp_src/core/nsselecter/nsselecter.cc

@@ -43,9 +43,8 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
 
 	bool needPutCachedTotal = false;
 	const auto initTotalCount = result.totalCount;
-	bool containAggCount = containSomeAggCount(AggCount);
-	bool containAggCountCached = containAggCount ? false : containSomeAggCount(AggCountCached);
-
+	const bool containAggCount = containSomeAggCount(AggCount);
+	const bool containAggCountCached = containAggCount ? false : containSomeAggCount(AggCountCached);
 	bool needCalcTotal = aggregationQueryRef.CalcTotal() == ModeAccurateTotal || containAggCount;
 
 	QueryCacheKey ckey;
@@ -100,6 +99,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
 	if (isFt) {
 		qPreproc.CheckUniqueFtQuery();
 		qPreproc.ExcludeFtQuery(rdxCtx);
+		result.haveRank = true;
 	}
 	qPreproc.ConvertWhereValues();
 
@@ -109,7 +109,6 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
 		result.addNSContext(ns_->payloadType_, ns_->tagsMatcher_, FieldsSet(ns_->tagsMatcher_, ctx.query.SelectFilters()), ns_->schema_);
 	}
 
-	if (isFt) result.haveRank = true;
 	if (ctx.query.IsWithRank()) {
 		if (isFt) {
 			result.needOutputRank = true;
@@ -321,16 +320,17 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
 			} else if (!reverse && !hasComparators && !aggregationsOnly) {
 				selectLoop<false, false, false>(lctx, result, rdxCtx);
 			}
-		}
 
-		// Get total count for simple query with 1 condition and 1 idset
-		if (needCalcTotal && !lctx.calcTotal) {
-			if (!ctx.query.Entries().Empty()) {
-				result.totalCount += qres.Get<SelectIterator>(0).GetMaxIterations();
-			} else {
-				result.totalCount += ns_->items_.size() - ns_->free_.size();
+			// Get total count for simple query with 1 condition and 1 idset
+			if (needCalcTotal && !lctx.calcTotal) {
+				if (!ctx.query.Entries().Empty()) {
+					result.totalCount += qres.Get<SelectIterator>(0).GetMaxIterations();
+				} else {
+					result.totalCount += ns_->items_.size() - ns_->free_.size();
+				}
 			}
 		}
+
 		explain.AddLoopTime();
 		explain.AddIterations(maxIterations);
 		if (!ctx.inTransaction) {
@@ -402,7 +402,9 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
 	}
 
 	if (needPutCachedTotal) {
-		logPrintf(LogTrace, "[%s] put totalCount value into query cache: %d ", ns_->name_, result.totalCount);
+		if rx_unlikely (logLevel >= LogTrace) {
+			logPrintf(LogInfo, "[%s] put totalCount value into query cache: %d ", ns_->name_, result.totalCount);
+		}
 		ns_->queryCountCache_->Put(ckey, {static_cast<size_t>(result.totalCount - initTotalCount)});
 	}
 	if (ctx.preResult && ctx.preResult->executionMode == JoinPreResult::ModeBuild) {
@@ -1018,7 +1020,7 @@ void NsSelecter::selectLoop(LoopCtx &ctx, ResultsT &result, const RdxContext &rd
 	SelectIterator &firstIterator = qres.begin()->Value<SelectIterator>();
 	IdType rowId = firstIterator.Val();
 	while (firstIterator.Next(rowId) && !finish) {
-		if (!sctx.inTransaction && (rowId % kCancelCheckFrequency == 0)) ThrowOnCancel(rdxCtx);
+		if ((rowId % kCancelCheckFrequency == 0) && !sctx.inTransaction) ThrowOnCancel(rdxCtx);
 		rowId = firstIterator.Val();
 		IdType properRowId = rowId;
 
@@ -1082,8 +1084,10 @@ void NsSelecter::selectLoop(LoopCtx &ctx, ResultsT &result, const RdxContext &rd
 						getSortIndexValue(sctx.sortingContext, properRowId, prevValues, proc,
 										  sctx.nsid < result.joined_.size() ? &result.joined_[sctx.nsid] : nullptr, joinedSelectors);
 				}
-				if (!ctx.count && !ctx.calcTotal && multiSortFinished) break;
-				if (ctx.calcTotal) result.totalCount++;
+				if (!ctx.count && !ctx.calcTotal && multiSortFinished) {
+					break;
+				}
+				result.totalCount += int(ctx.calcTotal);
 			} else {
 				assertf(static_cast<size_t>(properRowId) < result.rowId2Vdoc->size(),
 						"properRowId = %d; rowId = %d; result.rowId2Vdoc->size() = %d", properRowId, rowId, result.rowId2Vdoc->size());

cpp_src/core/nsselecter/querypreprocessor.cc

@@ -23,7 +23,8 @@ QueryPreprocessor::QueryPreprocessor(QueryEntries &&queries, NamespaceImpl *ns,
 	  start_(query_.Offset()),
 	  count_(query_.Limit()),
 	  forcedSortOrder_(!query_.forcedSortOrder_.empty()),
-	  reqMatchedOnce_(ctx.reqMatchedOnceFlag) {
+	  reqMatchedOnce_(ctx.reqMatchedOnceFlag),
+	  isMergeQuery_(ctx.isMergeQuery == IsMergeQuery::Yes) {
 	if (forcedSortOrder_ && (start_ > QueryEntry::kDefaultOffset || count_ < QueryEntry::kDefaultLimit)) {
 		assertrx_throw(!query_.sortingEntries_.empty());
 		static const std::vector<JoinedSelector> emptyJoinedSelectors;
@@ -40,7 +41,7 @@ QueryPreprocessor::QueryPreprocessor(QueryEntries &&queries, NamespaceImpl *ns,
 			queryEntryAddedByForcedSortOptimization_ = true;
 		}
 	}
-	if (ctx.isMergeQuery == IsMergeQuery::Yes) {
+	if (isMergeQuery_) {
 		if (QueryEntry::kDefaultLimit - start_ > count_) {
 			count_ += start_;
 		} else {
@@ -83,8 +84,17 @@ bool QueryPreprocessor::NeedNextEvaluation(unsigned start, unsigned count, bool
 	} else if (ftEntry_) {
 		if (!matchedAtLeastOnce) return false;
 		qresHolder.BackupContainer();
-		start_ = query_.Offset();
-		count_ = query_.Limit();
+		if (isMergeQuery_) {
+			if (QueryEntry::kDefaultLimit - query_.Offset() > query_.Limit()) {
+				count_ = query_.Limit() + query_.Offset();
+			} else {
+				count_ = QueryEntry::kDefaultLimit;
+			}
+			start_ = QueryEntry::kDefaultOffset;
+		} else {
+			start_ = query_.Offset();
+			count_ = query_.Limit();
+		}
 		forcedSortOrder_ = !query_.forcedSortOrder_.empty();
 		clear();
 		Append(OpAnd, std::move(*ftEntry_));

cpp_src/core/nsselecter/querypreprocessor.h

@@ -119,6 +119,7 @@ class QueryPreprocessor : private QueryEntries {
 	bool desc_ = false;
 	bool forcedSortOrder_ = false;
 	bool reqMatchedOnce_ = false;
+	const bool isMergeQuery_ = false;
 	std::optional<QueryEntry> ftEntry_;
 	std::optional<FtPreselectT> ftPreselect_;
 };

cpp_src/core/query/queryentry.cc

@@ -144,7 +144,7 @@ void VerifyQueryEntryValues(CondType cond, const VariantArray &values) {
 		case CondLike:
 			checkArgsCount(1);
 			if (!values[0].Type().Is<KeyValueType::String>()) {
-				throw Error{errLogic, "Condition %s must have string argument, but %d argument was provided", CondTypeToStr(cond),
+				throw Error{errLogic, "Condition %s must have string argument, but %s argument was provided", CondTypeToStr(cond),
 							values[0].Type().Name()};
 			}
 			break;

cpp_src/gtests/tests/CMakeLists.txt

@@ -56,7 +56,7 @@ endif ()
 
 set(GTEST_TIMEOUT 600)
 if (WITH_ASAN OR WITH_TSAN OR WITH_STDLIB_DEBUG)
-    set(GTEST_TIMEOUT 1200)
+    set(GTEST_TIMEOUT 2000)
 endif ()
 
 find_program(GTEST_PARALLEL "gtest-parallel")

cpp_src/gtests/tests/fixtures/ft_api.cc

@@ -1,5 +1,4 @@
 #include "ft_api.h"
-#include "core/cjson/jsonbuilder.h"
 
 void FTApi::Init(const reindexer::FtFastConfig& ftCfg, unsigned nses, const std::string& storage) {
 	rt.reindexer.reset(new reindexer::Reindexer);

cpp_src/gtests/tests/fixtures/reindexer_api.h

@@ -1,17 +1,8 @@
 #pragma once
 
-#include <gtest/gtest.h>
-#include <memory>
-#include <string>
-#include <tuple>
-
-#include <iostream>
-#include "core/keyvalue/variant.h"
-#include "core/query/query.h"
 #include "core/reindexer.h"
 #include "reindexertestapi.h"
 #include "servercontrol.h"
-#include "tools/errors.h"
 
 using reindexer::Error;
 using reindexer::Item;

cpp_src/gtests/tests/fixtures/replication_api.cc

@@ -1,10 +1,4 @@
 #include "replication_api.h"
-#include <fstream>
-#include <thread>
-#include "core/cjson/jsonbuilder.h"
-#include "core/dbconfig.h"
-#include "tools/fsops.h"
-#include "vendor/gason/gason.h"
 
 const std::string ReplicationApi::kConfigNs = "#config";
 
@@ -112,7 +106,7 @@ void ReplicationApi::SwitchMaster(size_t id, const ReplicationConfigTest::NsSet&
 	for (size_t i = 0; i < svc_.size(); i++) {
 		std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(kDefaultRpcPort + masterId_) + "/node" + std::to_string(masterId_);
 		ReplicationConfigTest config("slave", false, true, i, masterDsn, "server_" + std::to_string(i), namespaces);
-		if (i != masterId_) GetSrv(i)->MakeSlave(masterId_, config);
+		if (i != masterId_) GetSrv(i)->MakeSlave(config);
 	}
 }
 
@@ -148,7 +142,7 @@ void ReplicationApi::SetUp() {
 		} else {
 			std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(kDefaultRpcPort + 0) + "/node" + std::to_string(0);
 			ReplicationConfigTest config("slave", false, true, i, masterDsn);
-			svc_.back().Get()->MakeSlave(0, config);
+			svc_.back().Get()->MakeSlave(config);
 		}
 	}
 }