Merge branch 'unstable' into minor/trying-to-fix-sonar

.asf.yaml

@@ -53,6 +53,7 @@ github:
     '2.7': {}
     '2.8': {}
     '2.9': {}
+    '2.10': {}
 
 notifications:
   commits:      commits@kvrocks.apache.org

.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:noble
+RUN apt update \
+    && apt install -y \
+        git build-essential cmake libtool python3 libssl-dev python3-pip \
+        wget curl clang-format-14 clang-tidy-14 golang-go ninja-build \
+        redis-tools vim python3-redis redis-server clang lld mold gdb fish
+RUN BUILD_DIR=$(pwd) && git clone https://github.com/jsha/minica /opt/minica \
+    && cd /opt/minica && git checkout 96a5c93723cf3d34b50b3e723a9f05cd3765bc67 && go build && cd $BUILD_DIR \
+    && echo 'export PATH=/opt/minica:$PATH' >> $HOME/.bashrc

.devcontainer/devcontainer.json

@@ -0,0 +1,5 @@
+{
+  "build": {
+    "dockerfile": "Dockerfile"
+  }
+}

.github/config/typos.toml

@@ -20,6 +20,11 @@ extend-exclude = [
     ".git/",
     "src/vendor/",
     "tests/gocase/util/slot.go",
+
+    # Uses short strings for testing glob matching
+    "tests/cppunit/string_util_test.cc",
+    "tests/gocase/unit/keyspace/keyspace_test.go",
+    "tests/gocase/unit/scan/scan_test.go",
 ]
 ignore-hidden = false
 

.github/workflows/kvrocks.yaml

@@ -40,7 +40,7 @@ jobs:
       docs_only: ${{ steps.result.outputs.docs_only }}
     steps:
       - uses: actions/checkout@v4
-      - uses: dorny/paths-filter@v3.0.0
+      - uses: dorny/paths-filter@v3
         id: changes
         with:
           filters: .github/config/changes.yml
@@ -58,7 +58,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.22.9
+        uses: crate-ci/typos@v1.25.0
         with:
           config: .github/config/typos.toml
 
@@ -204,11 +204,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       SONARCLOUD_OUTPUT_DIR: sonarcloud-data
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true
     steps:
       - name: Setup macOS
         if: ${{ startsWith(matrix.os, 'macos') }}
         run: |
-          brew install cmake gcc autoconf automake libtool openssl coreutils
+          brew install --quiet --formula cmake gcc autoconf automake libtool openssl coreutils
           echo "NPROC=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV
           echo "CMAKE_EXTRA_DEFS=-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl" >> $GITHUB_ENV
       - name: Setup Linux
@@ -486,15 +487,15 @@ jobs:
 
       - name: Cache redis
         id: cache-redis
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/local/bin/redis-cli
           key: ${{ matrix.image }}-redis-cli
 
       - name: Cache redis server
         id: cache-redis-server
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/local/bin/redis-server
@@ -540,6 +541,8 @@ jobs:
         run: apt install -y python3-redis
 
       - name: Run kvrocks2redis Test
+        # FIXME: https://github.com/apache/kvrocks/issues/2574
+        if: ${{ !startsWith(matrix.image, 'archlinux') }}
         shell: bash
         run: |
           $HOME/local/bin/redis-server --daemonize yes

.github/workflows/sonar.yaml

@@ -34,7 +34,7 @@ jobs:
           ref: ${{ github.event.workflow_run.head_sha }}
           fetch-depth: 0
       - name: Install sonar-scanner and build-wrapper
-        uses: SonarSource/sonarcloud-github-c-cpp@v2
+        uses: SonarSource/sonarcloud-github-c-cpp@v3
       - name: 'Download code coverage'
         uses: actions/github-script@v7
         with:

README.md

@@ -24,8 +24,7 @@
 [![GitHub stars](https://img.shields.io/github/stars/apache/kvrocks)](https://github.com/apache/kvrocks/stargazers)
 
 ---
-
-* [Slack Channel](https://join.slack.com/t/kvrockscommunity/shared_invite/zt-p5928e3r-OUAK8SUgC8GOceGM6dAz6w)
+* [Chat on Zulip](https://kvrocks.zulipchat.com/)
 * [Mailing List](https://lists.apache.org/list.html?dev@kvrocks.apache.org) ([how to subscribe](https://www.apache.org/foundation/mailinglists.html#subscribing))
 
 **Apache Kvrocks** is a distributed key value NoSQL database that uses RocksDB as storage engine and is compatible with Redis protocol. Kvrocks intends to decrease the cost of memory and increase the capacity while compared to Redis. The design of replication and storage was inspired by [rocksplicator](https://github.com/pinterest/rocksplicator) and [blackwidow](https://github.com/Qihoo360/blackwidow).

cmake/cpptrace.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(cpptrace
-  jeremy-rifkin/cpptrace v0.7.0
-  MD5=d897c48f5bf96134109f7e6716f2fd31
+  jeremy-rifkin/cpptrace v0.7.2
+  MD5=4d992a22ddb80300fa2ddac097a5ce51
 )
 
 if (SYMBOLIZE_BACKEND STREQUAL "libbacktrace")

cmake/jsoncons.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(jsoncons
-  danielaparker/jsoncons v0.176.0
-  MD5=5d0343fe48cbc640bdb42d89a5b87182
+  danielaparker/jsoncons v0.178.0
+  MD5=397410843b7c540e9dcee9b8b0c797a6
 )
 
 FetchContent_MakeAvailableWithArgs(jsoncons

cmake/pegtl.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubTarWithMirror(pegtl
-  taocpp/PEGTL 3.2.7
-  MD5=31b14660c883bc0489ddcdfbd29199c9
+  taocpp/PEGTL 3.2.8
+  MD5=50339029d1bb037909b28c382214033e
 )
 
 FetchContent_MakeAvailableWithArgs(pegtl)

cmake/rocksdb.cmake

@@ -26,8 +26,8 @@ endif()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(rocksdb
-  facebook/rocksdb v9.6.1
-  MD5=ce31144a7e65d8f4f3f9d98986509eb1
+  facebook/rocksdb v9.7.4
+  MD5=9a08feb50e017006146bcff37059096f
 )
 
 FetchContent_GetProperties(jemalloc)

cmake/tbb.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(tbb
-  oneapi-src/oneTBB v2021.13.0
-  MD5=2dd9b7cfa5de5bb3add2f7392e0c9bab
+  oneapi-src/oneTBB v2022.0.0
+  MD5=7eaeff0ddec85182afb60f2232fae2af
 )
 
 FetchContent_MakeAvailableWithArgs(tbb

kvrocks.conf

@@ -20,6 +20,16 @@ bind 127.0.0.1
 # unixsocket /tmp/kvrocks.sock
 # unixsocketperm 777
 
+# Allows a parent process to open a socket and pass its FD down to kvrocks as a child 
+# process. Useful to reserve a port and prevent race conditions.
+# 
+# PLEASE NOTE: 
+# If this is overridden to a value other than -1, the bind and tls* directives will be 
+# ignored.
+# 
+# Default: -1 (not overridden, defer to creating a connection to the specified port)
+socket-fd -1
+
 # Accept connections on the specified port, default is 6666.
 port 6666
 
@@ -708,6 +718,16 @@ rocksdb.max_background_flushes -1
 # Default: 2
 rocksdb.max_subcompactions 2
 
+# If enabled WAL records will be compressed before they are written. Only
+# ZSTD (= kZSTD) is supported (until streaming support is adapted for other
+# compression types). Compressed WAL records will be read in supported
+# versions (>= RocksDB 7.4.0 for ZSTD) regardless of this setting when
+# the WAL is read.
+#
+# Accept value: "no", "zstd"
+# Default is no
+rocksdb.wal_compression no
+
 # In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size
 # as the trigger of column family flush. Once WALs exceed this size, RocksDB
 # will start forcing the flush of column families to allow deletion of some
@@ -794,6 +814,15 @@ rocksdb.compression_level 32767
 # Default: 2 MB
 rocksdb.compaction_readahead_size 2097152
 
+# Enable compression from n levels of LSM-tree.
+# By default compression is disabled for the first two levels (L0 and L1),
+# because it may contain the frequently accessed data, so it'd be better
+# to use uncompressed data to save the CPU.
+# Value: [0, 7) (upper boundary is kvrocks maximum levels number)
+#
+# Default: 2
+rocksdb.compression_start_level 2
+
 # he limited write rate to DB if soft_pending_compaction_bytes_limit or
 # level0_slowdown_writes_trigger is triggered.
 

src/cli/main.cc

@@ -30,7 +30,6 @@
 #include <iomanip>
 #include <ostream>
 
-#include "config.h"
 #include "daemon_util.h"
 #include "io_util.h"
 #include "pid_util.h"
@@ -40,9 +39,7 @@
 #include "storage/storage.h"
 #include "string_util.h"
 #include "time_util.h"
-#include "unique_fd.h"
 #include "vendor/crc64.h"
-#include "version.h"
 #include "version_util.h"
 
 Server *srv = nullptr;
@@ -136,14 +133,19 @@ int main(int argc, char *argv[]) {
     std::cout << "Failed to load config. Error: " << s.Msg() << std::endl;
     return 1;
   }
+  const auto socket_fd_exit = MakeScopeExit([&config] {
+    if (config.socket_fd != -1) {
+      close(config.socket_fd);
+    }
+  });
 
   crc64_init();
   InitGoogleLog(&config);
   LOG(INFO) << "kvrocks " << PrintVersion;
   // Tricky: We don't expect that different instances running on the same port,
   // but the server use REUSE_PORT to support the multi listeners. So we connect
   // the listen port to check if the port has already listened or not.
-  if (!config.binds.empty()) {
+  if (config.socket_fd == -1 && !config.binds.empty()) {
     uint32_t ports[] = {config.port, config.tls_port, 0};
     for (uint32_t *port = ports; *port; ++port) {
       if (util::IsPortInUse(*port)) {

src/cluster/cluster.cc

@@ -75,9 +75,7 @@ Status Cluster::SetNodeId(const std::string &node_id) {
   }
 
   // Set replication relationship
-  if (myself_) return SetMasterSlaveRepl();
-
-  return Status::OK();
+  return SetMasterSlaveRepl();
 }
 
 // The reason why the new version MUST be +1 of current version is that,
@@ -204,11 +202,8 @@ Status Cluster::SetClusterNodes(const std::string &nodes_str, int64_t version, b
   }
 
   // Set replication relationship
-  if (myself_) {
-    s = SetMasterSlaveRepl();
-    if (!s.IsOK()) {
-      return s.Prefixed("failed to set master-replica replication");
-    }
+  if (auto s = SetMasterSlaveRepl(); !s.IsOK()) {
+    return s.Prefixed("failed to set master-replica replication");
   }
 
   // Clear data of migrated slots
@@ -234,7 +229,13 @@ Status Cluster::SetClusterNodes(const std::string &nodes_str, int64_t version, b
 Status Cluster::SetMasterSlaveRepl() {
   if (!srv_) return Status::OK();
 
-  if (!myself_) return Status::OK();
+  // If the node is not in the cluster topology, remove the master replication if it's a replica.
+  if (!myself_) {
+    if (auto s = srv_->RemoveMaster(); !s.IsOK()) {
+      return s.Prefixed("failed to remove master");
+    }
+    return Status::OK();
+  }
 
   if (myself_->role == kClusterMaster) {
     // Master mode
@@ -773,9 +774,14 @@ Status Cluster::parseClusterNodes(const std::string &nodes_str, ClusterNodes *no
 
     // 6) slot info
     auto valid_range = NumericRange<int>{0, kClusterSlots - 1};
+    const std::regex node_id_regex(R"(\b[a-fA-F0-9]{40}\b)");
     for (unsigned i = 5; i < fields.size(); i++) {
       std::vector<std::string> ranges = util::Split(fields[i], "-");
       if (ranges.size() == 1) {
+        if (std::regex_match(fields[i], node_id_regex)) {
+          return {Status::ClusterInvalidInfo, "Invalid nodes definition: Missing newline between node entries."};
+        }
+
         auto parse_start = ParseInt<int>(ranges[0], valid_range, 10);
         if (!parse_start) {
           return {Status::ClusterInvalidInfo, errSlotOutOfRange};
@@ -827,16 +833,16 @@ bool Cluster::IsWriteForbiddenSlot(int slot) const {
 
 Status Cluster::CanExecByMySelf(const redis::CommandAttributes *attributes, const std::vector<std::string> &cmd_tokens,
                                 redis::Connection *conn, lua::ScriptRunCtx *script_run_ctx) {
-  std::vector<int> keys_indexes;
+  std::vector<int> key_indexes;
 
-  // No keys
-  if (auto s = redis::CommandTable::GetKeysFromCommand(attributes, cmd_tokens, &keys_indexes); !s.IsOK())
-    return Status::OK();
+  auto s = redis::CommandTable::GetKeysFromCommand(attributes, cmd_tokens);
+  if (!s) return Status::OK();
+  key_indexes = *s;
 
-  if (keys_indexes.empty()) return Status::OK();
+  if (key_indexes.empty()) return Status::OK();
 
   int slot = -1;
-  for (auto i : keys_indexes) {
+  for (auto i : key_indexes) {
     if (i >= static_cast<int>(cmd_tokens.size())) break;
 
     int cur_slot = GetSlotIdFromKey(cmd_tokens[i]);
@@ -866,6 +872,8 @@ Status Cluster::CanExecByMySelf(const redis::CommandAttributes *attributes, cons
     cross_slot_ok = true;
   }
 
+  uint64_t flags = attributes->GenerateFlags(cmd_tokens);
+
   if (myself_ && myself_ == slots_nodes_[slot]) {
     // We use central controller to manage the topology of the cluster.
     // Server can't change the topology directly, so we record the migrated slots
@@ -875,7 +883,7 @@ Status Cluster::CanExecByMySelf(const redis::CommandAttributes *attributes, cons
     }
     // To keep data consistency, slot will be forbidden write while sending the last incremental data.
     // During this phase, the requests of the migrating slot has to be rejected.
-    if ((attributes->flags & redis::kCmdWrite) && IsWriteForbiddenSlot(slot)) {
+    if ((flags & redis::kCmdWrite) && IsWriteForbiddenSlot(slot)) {
       return {Status::RedisTryAgain, "Can't write to slot being migrated which is in write forbidden phase"};
     }
 
@@ -898,7 +906,7 @@ Status Cluster::CanExecByMySelf(const redis::CommandAttributes *attributes, cons
     return Status::OK();  // I'm serving the imported slot
   }
 
-  if (myself_ && myself_->role == kClusterSlave && !(attributes->flags & redis::kCmdWrite) &&
+  if (myself_ && myself_->role == kClusterSlave && !(flags & redis::kCmdWrite) &&
       nodes_.find(myself_->master_id) != nodes_.end() && nodes_[myself_->master_id] == slots_nodes_[slot] &&
       conn->IsFlagEnabled(redis::Connection::kReadOnly)) {
     return Status::OK();  // My master is serving this slot