From 303574e4639275d23e7a307d6f9b63bcd8465762 Mon Sep 17 00:00:00 2001 From: Myth Date: Tue, 30 Jan 2024 21:07:51 +0800 Subject: [PATCH] Ignore max-db-size limit when deleting data or writing aux informations (#2047) --- src/commands/cmd_hash.cc | 2 +- src/commands/cmd_json.cc | 6 +++--- src/commands/cmd_key.cc | 4 ++-- src/commands/cmd_list.cc | 4 ++-- src/commands/cmd_server.cc | 20 ++++++++++++++++---- src/commands/cmd_set.cc | 2 +- src/commands/cmd_sortedint.cc | 2 +- src/commands/cmd_stream.cc | 4 ++-- src/commands/cmd_string.cc | 2 +- src/commands/cmd_zset.cc | 8 ++++---- src/commands/commander.h | 27 +++++++++++++++------------ src/server/redis_connection.cc | 5 +++++ src/storage/storage.cc | 4 ---- src/storage/storage.h | 4 +++- tests/gocase/unit/debug/debug_test.go | 24 ++++++++++++++++++++++++ 15 files changed, 80 insertions(+), 38 deletions(-) diff --git a/src/commands/cmd_hash.cc b/src/commands/cmd_hash.cc index cc4c475ebad..6db97f89025 100644 --- a/src/commands/cmd_hash.cc +++ b/src/commands/cmd_hash.cc @@ -439,7 +439,7 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("hget", 3, "read-only", 1, 1, 1 MakeCmdAttr("hincrbyfloat", 4, "write", 1, 1, 1), MakeCmdAttr("hset", -4, "write", 1, 1, 1), MakeCmdAttr("hsetnx", -4, "write", 1, 1, 1), - MakeCmdAttr("hdel", -3, "write", 1, 1, 1), + MakeCmdAttr("hdel", -3, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("hstrlen", 3, "read-only", 1, 1, 1), MakeCmdAttr("hexists", 3, "read-only", 1, 1, 1), MakeCmdAttr("hlen", 2, "read-only", 1, 1, 1), diff --git a/src/commands/cmd_json.cc b/src/commands/cmd_json.cc index 8cd49c51e3f..54a28271eae 100644 --- a/src/commands/cmd_json.cc +++ b/src/commands/cmd_json.cc @@ -600,15 +600,15 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("json.set", 4, "write", 1, 1 MakeCmdAttr("json.type", -2, "read-only", 1, 1, 1), MakeCmdAttr("json.arrappend", -4, "write", 1, 1, 1), MakeCmdAttr("json.arrinsert", -5, "write", 1, 1, 1), - MakeCmdAttr("json.arrtrim", 5, "write", 1, 1, 1), - MakeCmdAttr("json.clear", -2, "write", 1, 1, 1), + MakeCmdAttr("json.arrtrim", 5, "write no-dbsize-check", 1, 1, 1), + MakeCmdAttr("json.clear", -2, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("json.toggle", -2, "write", 1, 1, 1), MakeCmdAttr("json.arrlen", -2, "read-only", 1, 1, 1), MakeCmdAttr("json.merge", 4, "write", 1, 1, 1), MakeCmdAttr("json.objkeys", -2, "read-only", 1, 1, 1), MakeCmdAttr("json.arrpop", -2, "write", 1, 1, 1), MakeCmdAttr("json.arrindex", -4, "read-only", 1, 1, 1), - MakeCmdAttr("json.del", -2, "write", 1, 1, 1), + MakeCmdAttr("json.del", -2, "write no-dbsize-check", 1, 1, 1), // JSON.FORGET is an alias for JSON.DEL, refer: https://redis.io/commands/json.forget/ MakeCmdAttr("json.forget", -2, "write", 1, 1, 1), MakeCmdAttr("json.numincrby", 4, "write", 1, 1, 1), diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index f94f87fecf5..2eacdd1ef74 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -350,8 +350,8 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("ttl", 2, "read-only", 1, 1, 1), MakeCmdAttr("pexpireat", 3, "write", 1, 1, 1), MakeCmdAttr("expiretime", 2, "read-only", 1, 1, 1), MakeCmdAttr("pexpiretime", 2, "read-only", 1, 1, 1), - MakeCmdAttr("del", -2, "write", 1, -1, 1), - MakeCmdAttr("unlink", -2, "write", 1, -1, 1), + MakeCmdAttr("del", -2, "write no-dbsize-check", 1, -1, 1), + MakeCmdAttr("unlink", -2, "write no-dbsize-check", 1, -1, 1), MakeCmdAttr("rename", 3, "write", 1, 2, 1), MakeCmdAttr("renamenx", 3, "write", 1, 2, 1), ) diff --git a/src/commands/cmd_list.cc b/src/commands/cmd_list.cc index 726d2a70889..f354d64cc4b 100644 --- a/src/commands/cmd_list.cc +++ b/src/commands/cmd_list.cc @@ -861,9 +861,9 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("blpop", -3, "write no-script" MakeCmdAttr("lpush", -3, "write", 1, 1, 1), MakeCmdAttr("lpushx", -3, "write", 1, 1, 1), MakeCmdAttr("lrange", 4, "read-only", 1, 1, 1), - MakeCmdAttr("lrem", 4, "write", 1, 1, 1), + MakeCmdAttr("lrem", 4, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("lset", 4, "write", 1, 1, 1), - MakeCmdAttr("ltrim", 4, "write", 1, 1, 1), + MakeCmdAttr("ltrim", 4, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("lmpop", -4, "write", CommandLMPop::keyRangeGen), MakeCmdAttr("rpop", -2, "write", 1, 1, 1), MakeCmdAttr("rpoplpush", 3, "write", 1, 2, 1), diff --git a/src/commands/cmd_server.cc b/src/commands/cmd_server.cc index 10cbc8cdc4e..e8b5f78d6c6 100644 --- a/src/commands/cmd_server.cc +++ b/src/commands/cmd_server.cc @@ -594,8 +594,16 @@ class CommandDebug : public Commander { } else if (subcommand_ == "protocol" && args.size() == 3) { protocol_type_ = util::ToLower(args[2]); return Status::OK(); + } else if (subcommand_ == "dbsize-limit" && args.size() == 3) { + auto val = ParseInt(args[2], {0, 1}, 10); + if (!val) { + return {Status::RedisParseErr, "invalid debug dbsize-limit value"}; + } + + dbsize_limit_ = static_cast(val); + return Status::OK(); } - return {Status::RedisInvalidCmd, "Syntax error, DEBUG SLEEP |PROTOCOL "}; + return {Status::RedisInvalidCmd, "Syntax error, DEBUG SLEEP |PROTOCOL |DBSIZE-LIMIT <0|1>"}; } Status Execute(Server *srv, Connection *conn, std::string *output) override { @@ -638,8 +646,11 @@ class CommandDebug : public Commander { "Wrong protocol type name. Please use one of the following: " "string|integer|double|array|set|bignum|true|false|null"); } + } else if (subcommand_ == "dbsize-limit") { + srv->storage->SetDBSizeLimit(dbsize_limit_); + *output = redis::SimpleString("OK"); } else { - return {Status::RedisInvalidCmd, "Unknown subcommand, should be DEBUG or PROTOCOL"}; + return {Status::RedisInvalidCmd, "Unknown subcommand, should be DEBUG, PROTOCOL or DBSIZE-LIMIT"}; } return Status::OK(); } @@ -648,6 +659,7 @@ class CommandDebug : public Commander { std::string subcommand_; std::string protocol_type_; uint64_t microsecond_ = 0; + bool dbsize_limit_ = false; }; class CommandCommand : public Commander { @@ -1318,8 +1330,8 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("auth", 2, "read-only ok-loadin MakeCmdAttr("config", -2, "read-only", 0, 0, 0, GenerateConfigFlag), MakeCmdAttr("namespace", -3, "read-only exclusive", 0, 0, 0), MakeCmdAttr("keys", 2, "read-only", 0, 0, 0), - MakeCmdAttr("flushdb", 1, "write", 0, 0, 0), - MakeCmdAttr("flushall", 1, "write", 0, 0, 0), + MakeCmdAttr("flushdb", 1, "write no-dbsize-check", 0, 0, 0), + MakeCmdAttr("flushall", 1, "write no-dbsize-check", 0, 0, 0), MakeCmdAttr("dbsize", -1, "read-only", 0, 0, 0), MakeCmdAttr("slowlog", -2, "read-only", 0, 0, 0), MakeCmdAttr("perflog", -2, "read-only", 0, 0, 0), diff --git a/src/commands/cmd_set.cc b/src/commands/cmd_set.cc index ced252234b2..213a6768279 100644 --- a/src/commands/cmd_set.cc +++ b/src/commands/cmd_set.cc @@ -438,7 +438,7 @@ class CommandSScan : public CommandSubkeyScanBase { }; REDIS_REGISTER_COMMANDS(MakeCmdAttr("sadd", -3, "write", 1, 1, 1), - MakeCmdAttr("srem", -3, "write", 1, 1, 1), + MakeCmdAttr("srem", -3, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("scard", 2, "read-only", 1, 1, 1), MakeCmdAttr("smembers", 2, "read-only", 1, 1, 1), MakeCmdAttr("sismember", 3, "read-only", 1, 1, 1), diff --git a/src/commands/cmd_sortedint.cc b/src/commands/cmd_sortedint.cc index b668a0a69e6..a97e357f154 100644 --- a/src/commands/cmd_sortedint.cc +++ b/src/commands/cmd_sortedint.cc @@ -250,7 +250,7 @@ class CommandSortedintRevRangeByValue : public CommandSortedintRangeByValue { }; REDIS_REGISTER_COMMANDS(MakeCmdAttr("siadd", -3, "write", 1, 1, 1), - MakeCmdAttr("sirem", -3, "write", 1, 1, 1), + MakeCmdAttr("sirem", -3, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("sicard", 2, "read-only", 1, 1, 1), MakeCmdAttr("siexists", -3, "read-only", 1, 1, 1), MakeCmdAttr("sirange", -4, "read-only", 1, 1, 1), diff --git a/src/commands/cmd_stream.cc b/src/commands/cmd_stream.cc index 7ba408859c1..76e2146b7b0 100644 --- a/src/commands/cmd_stream.cc +++ b/src/commands/cmd_stream.cc @@ -1190,14 +1190,14 @@ class CommandXSetId : public Commander { }; REDIS_REGISTER_COMMANDS(MakeCmdAttr("xadd", -5, "write", 1, 1, 1), - MakeCmdAttr("xdel", -3, "write", 1, 1, 1), + MakeCmdAttr("xdel", -3, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("xgroup", -4, "write", 2, 2, 1), MakeCmdAttr("xlen", -2, "read-only", 1, 1, 1), MakeCmdAttr("xinfo", -2, "read-only", 0, 0, 0), MakeCmdAttr("xrange", -4, "read-only", 1, 1, 1), MakeCmdAttr("xrevrange", -2, "read-only", 1, 1, 1), MakeCmdAttr("xread", -4, "read-only", 0, 0, 0), - MakeCmdAttr("xtrim", -4, "write", 1, 1, 1), + MakeCmdAttr("xtrim", -4, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("xsetid", -3, "write", 1, 1, 1)) } // namespace redis diff --git a/src/commands/cmd_string.cc b/src/commands/cmd_string.cc index debf5e3fb7b..3f7b5090b8c 100644 --- a/src/commands/cmd_string.cc +++ b/src/commands/cmd_string.cc @@ -626,7 +626,7 @@ REDIS_REGISTER_COMMANDS( MakeCmdAttr("getset", 3, "write", 1, 1, 1), MakeCmdAttr("getrange", 4, "read-only", 1, 1, 1), MakeCmdAttr("substr", 4, "read-only", 1, 1, 1), - MakeCmdAttr("getdel", 2, "write", 1, 1, 1), + MakeCmdAttr("getdel", 2, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("setrange", 4, "write", 1, 1, 1), MakeCmdAttr("mget", -2, "read-only", 1, -1, 1), MakeCmdAttr("append", 3, "write", 1, 1, 1), MakeCmdAttr("set", -3, "write", 1, 1, 1), diff --git a/src/commands/cmd_zset.cc b/src/commands/cmd_zset.cc index d78fe54eed8..136b84eb5e2 100644 --- a/src/commands/cmd_zset.cc +++ b/src/commands/cmd_zset.cc @@ -1537,10 +1537,10 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("zadd", -4, "write", 1, 1, 1), MakeCmdAttr("zrevrangebylex", -4, "read-only", 1, 1, 1), MakeCmdAttr("zrangebyscore", -4, "read-only", 1, 1, 1), MakeCmdAttr("zrank", -3, "read-only", 1, 1, 1), - MakeCmdAttr("zrem", -3, "write", 1, 1, 1), - MakeCmdAttr("zremrangebyrank", 4, "write", 1, 1, 1), - MakeCmdAttr("zremrangebyscore", 4, "write", 1, 1, 1), - MakeCmdAttr("zremrangebylex", 4, "write", 1, 1, 1), + MakeCmdAttr("zrem", -3, "write no-dbsize-check", 1, 1, 1), + MakeCmdAttr("zremrangebyrank", 4, "write no-dbsize-check", 1, 1, 1), + MakeCmdAttr("zremrangebyscore", 4, "write no-dbsize-check", 1, 1, 1), + MakeCmdAttr("zremrangebylex", 4, "write no-dbsize-check", 1, 1, 1), MakeCmdAttr("zrevrangebyscore", -4, "read-only", 1, 1, 1), MakeCmdAttr("zrevrank", -3, "read-only", 1, 1, 1), MakeCmdAttr("zscore", 3, "read-only", 1, 1, 1), diff --git a/src/commands/commander.h b/src/commands/commander.h index 3330337c746..85cdde7651d 100644 --- a/src/commands/commander.h +++ b/src/commands/commander.h @@ -52,18 +52,19 @@ class Connection; struct CommandAttributes; enum CommandFlags : uint64_t { - kCmdWrite = 1ULL << 0, // "write" flag - kCmdReadOnly = 1ULL << 1, // "read-only" flag - kCmdReplication = 1ULL << 2, // "replication" flag - kCmdPubSub = 1ULL << 3, // "pub-sub" flag - kCmdScript = 1ULL << 4, // "script" flag - kCmdLoading = 1ULL << 5, // "ok-loading" flag - kCmdMulti = 1ULL << 6, // "multi" flag - kCmdExclusive = 1ULL << 7, // "exclusive" flag - kCmdNoMulti = 1ULL << 8, // "no-multi" flag - kCmdNoScript = 1ULL << 9, // "no-script" flag - kCmdROScript = 1ULL << 10, // "ro-script" flag for read-only script commands - kCmdCluster = 1ULL << 11, // "cluster" flag + kCmdWrite = 1ULL << 0, // "write" flag + kCmdReadOnly = 1ULL << 1, // "read-only" flag + kCmdReplication = 1ULL << 2, // "replication" flag + kCmdPubSub = 1ULL << 3, // "pub-sub" flag + kCmdScript = 1ULL << 4, // "script" flag + kCmdLoading = 1ULL << 5, // "ok-loading" flag + kCmdMulti = 1ULL << 6, // "multi" flag + kCmdExclusive = 1ULL << 7, // "exclusive" flag + kCmdNoMulti = 1ULL << 8, // "no-multi" flag + kCmdNoScript = 1ULL << 9, // "no-script" flag + kCmdROScript = 1ULL << 10, // "ro-script" flag for read-only script commands + kCmdCluster = 1ULL << 11, // "cluster" flag + kCmdNoDBSizeCheck = 1ULL << 12, // "no-dbsize-check" flag }; class Commander { @@ -178,6 +179,8 @@ inline uint64_t ParseCommandFlags(const std::string &description, const std::str flags |= kCmdROScript; else if (flag == "cluster") flags |= kCmdCluster; + else if (flag == "no-dbsize-check") + flags |= kCmdNoDBSizeCheck; else { std::cout << fmt::format("Encountered non-existent flag '{}' in command {} in command attribute parsing", flag, cmd_name) diff --git a/src/server/redis_connection.cc b/src/server/redis_connection.cc index 99d579fe136..aa830ac8791 100644 --- a/src/server/redis_connection.cc +++ b/src/server/redis_connection.cc @@ -501,6 +501,11 @@ void Connection::ExecuteCommands(std::deque *to_process_cmds) { continue; } + if ((cmd_flags & kCmdWrite) && !(cmd_flags & kCmdNoDBSizeCheck) && srv_->storage->ReachedDBSizeLimit()) { + Reply(redis::Error("ERR write command not allowed when reached max-db-size.")); + continue; + } + if (!config->slave_serve_stale_data && srv_->IsSlave() && cmd_name != "info" && cmd_name != "slaveof" && srv_->GetReplicationState() != kReplConnected) { Reply( diff --git a/src/storage/storage.cc b/src/storage/storage.cc index 75f6fb6de86..0408c51ca7e 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -639,10 +639,6 @@ rocksdb::Status Storage::Write(const rocksdb::WriteOptions &options, rocksdb::Wr } rocksdb::Status Storage::writeToDB(const rocksdb::WriteOptions &options, rocksdb::WriteBatch *updates) { - if (db_size_limit_reached_) { - return rocksdb::Status::SpaceLimit(); - } - // Put replication id logdata at the end of write batch if (replid_.length() == kReplIdLength) { updates->PutLogData(ServerLogData(kReplIdLog, replid_).Encode()); diff --git a/src/storage/storage.h b/src/storage/storage.h index 6114c7fc437..7e0e94d2fc1 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -182,6 +182,8 @@ class Storage { void PurgeOldBackups(uint32_t num_backups_to_keep, uint32_t backup_max_keep_hours); uint64_t GetTotalSize(const std::string &ns = kDefaultNamespace); void CheckDBSizeLimit(); + bool ReachedDBSizeLimit() { return db_size_limit_reached_; } + void SetDBSizeLimit(bool limit) { db_size_limit_reached_ = limit; } void SetIORateLimit(int64_t max_io_mb); std::shared_lock ReadLockGuard(); @@ -254,7 +256,7 @@ class Storage { Config *config_ = nullptr; std::vector cf_handles_; LockManager lock_mgr_; - bool db_size_limit_reached_ = false; + std::atomic db_size_limit_reached_{false}; DBStats db_stats_; diff --git a/tests/gocase/unit/debug/debug_test.go b/tests/gocase/unit/debug/debug_test.go index 416e4a7d555..c15fd33dc2f 100644 --- a/tests/gocase/unit/debug/debug_test.go +++ b/tests/gocase/unit/debug/debug_test.go @@ -119,3 +119,27 @@ func TestDebugProtocolV3(t *testing.T) { require.EqualValues(t, false, val) }) } + +func TestDebugDBSizeLimit(t *testing.T) { + srv := util.StartServer(t, map[string]string{}) + defer srv.Close() + + ctx := context.Background() + rdb := srv.NewClient() + defer func() { require.NoError(t, rdb.Close()) }() + + t.Run("debug ignore dbsize check", func(t *testing.T) { + r := rdb.Do(ctx, "SET", "k1", "v1") + require.NoError(t, r.Err()) + + r = rdb.Do(ctx, "DEBUG", "DBSIZE-LIMIT", "1") + require.NoError(t, r.Err()) + + r = rdb.Do(ctx, "SET", "k2", "v2") + require.Error(t, r.Err()) + util.ErrorRegexp(t, r.Err(), "ERR.*not allowed.*") + + r = rdb.Do(ctx, "DEL", "k1") + require.NoError(t, r.Err()) + }) +}