Skip to content

Commit

Permalink
feat(log): Redacted user key-values printed in logs (#1598)
Browse files Browse the repository at this point in the history 
#1575

User key-values will be redacted if encryption enabled.
  • Loading branch information
@Samunroyu
Samunroyu authored Oct 18, 2023
1 parent 0f92716 commit 13a45ab
Show file tree
Hide file tree
Showing 8 changed files with 183 additions and 88 deletions.
23 changes: 23 additions & 0 deletions src/base/pegasus_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@
#include <string>
#include <vector>

#include "utils/flags.h"
#include "utils/string_view.h"

DSN_DECLARE_bool(encrypt_data_at_rest);

namespace dsn {
class rpc_address;
} // namespace dsn
Expand All @@ -40,6 +43,7 @@ namespace utils {
// it's seconds since 2016.01.01-00:00:00 GMT
const uint32_t epoch_begin = 1451606400;
inline uint32_t epoch_now() { return time(nullptr) - epoch_begin; }
const static std::string kRedactedString = "<redacted>";

// extract "host" from rpc_address
void addr2host(const ::dsn::rpc_address &addr, char *str, int len);
Expand Down Expand Up @@ -97,6 +101,15 @@ std::string c_escape_string(const T &src, bool always_escape = false)
return s;
}

template <class T>
std::string c_escape_sensitive_string(const T &src, bool always_escape = false)
{
if (FLAGS_encrypt_data_at_rest) {
return kRedactedString;
}
return c_escape_string(src, always_escape);
}

// ----------------------------------------------------------------------
// c_unescape_string()
// Copies 'src' to 'dest', unescaping '0xFF'-style escape sequences to
Expand All @@ -106,6 +119,16 @@ std::string c_escape_string(const T &src, bool always_escape = false)
// ----------------------------------------------------------------------
int c_unescape_string(const std::string &src, std::string &dest);

template <class T>
const std::string &redact_sensitive_string(const T &src)
{
if (FLAGS_encrypt_data_at_rest) {
return kRedactedString;
} else {
return src;
}
}

inline dsn::string_view to_string_view(rocksdb::Slice s) { return {s.data(), s.size()}; }

inline rocksdb::Slice to_rocksdb_slice(dsn::string_view s) { return {s.data(), s.size()}; }
Expand Down
69 changes: 69 additions & 0 deletions src/base/test/redact_sensitive_string_test.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

// IWYU pragma: no_include <gtest/gtest-message.h>
// IWYU pragma: no_include <gtest/gtest-test-part.h>
#include <gtest/gtest.h>
#include <string>

#include "base/pegasus_utils.h"

const std::string test_string = "pegasus";

TEST(pegasus_utils, redact_sensitive_string)
{
FLAGS_encrypt_data_at_rest = true;
auto result_string = pegasus::utils::redact_sensitive_string(test_string);
ASSERT_EQ(pegasus::utils::kRedactedString, result_string);
}

TEST(pegasus_utils, redact_sensitive_string_with_encrypt)
{
FLAGS_encrypt_data_at_rest = false;
auto result_string = pegasus::utils::redact_sensitive_string(test_string);
ASSERT_EQ("pegasus", result_string);
}

TEST(pegasus_utils, c_escape_sensitive_string_with_no_encrypt_and_escape)
{
FLAGS_encrypt_data_at_rest = false;
auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, false);
ASSERT_EQ("pegasus", result_string);
}

TEST(pegasus_utils, c_escape_sensitive_string_with_no_encrypt)
{
FLAGS_encrypt_data_at_rest = false;
auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, true);
ASSERT_EQ("\\x70\\x65\\x67\\x61\\x73\\x75\\x73", result_string);
}

TEST(pegasus_utils, c_escape_sensitive_string_with_encrypt)
{
FLAGS_encrypt_data_at_rest = true;
auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, false);
ASSERT_EQ(pegasus::utils::kRedactedString, result_string);
}

TEST(pegasus_utils, c_escape_sensitive_string_with_encrypt_and_escape)
{
FLAGS_encrypt_data_at_rest = true;
auto result_string = pegasus::utils::c_escape_sensitive_string(test_string, true);
ASSERT_EQ(pegasus::utils::kRedactedString, result_string);
}
59 changes: 30 additions & 29 deletions src/geo/lib/geo_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ int geo_client::set(const std::string &hash_key,
auto async_set_callback = [&](int ec_, pegasus_client::internal_info &&info_) {
if (ec_ != PERR_OK) {
LOG_ERROR("set data failed. hash_key={}, sort_key={}, error={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
ret = ec_;
}
Expand Down Expand Up @@ -186,8 +186,8 @@ void geo_client::async_set(const std::string &hash_key,
if (ec_ != PERR_OK) {
LOG_ERROR("set {} data failed. hash_key={}, sort_key={}, error={}",
data_type_ == DataType::common ? "common" : "geo",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
*ret = ec_;
}
Expand Down Expand Up @@ -238,8 +238,8 @@ int geo_client::get(const std::string &hash_key,
lng_degrees = lng_degrees_;
} else {
LOG_WARNING("get data failed. hash_key={}, sort_key={}, error={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
}
ret = ec_;
Expand Down Expand Up @@ -269,8 +269,8 @@ void geo_client::async_get(const std::string &hash_key,
S2LatLng latlng;
if (!_codec.decode_from_value(value_, latlng)) {
LOG_ERROR("decode_from_value failed. hash_key={}, sort_key={}, value={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
value_);
cb(PERR_GEO_DECODE_VALUE_ERROR, id, 0, 0);
return;
Expand All @@ -290,8 +290,8 @@ int geo_client::del(const std::string &hash_key,
auto async_del_callback = [&](int ec_, pegasus_client::internal_info &&info_) {
if (ec_ != PERR_OK) {
LOG_ERROR("del data failed. hash_key={}, sort_key={}, error={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
ret = ec_;
}
Expand Down Expand Up @@ -358,8 +358,8 @@ void geo_client::async_del(const std::string &hash_key,
if (ec__ != PERR_OK) {
LOG_ERROR("del {} data failed. hash_key={}, sort_key={}, error={}",
data_type_ == DataType::common ? "common" : "geo",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
*ret = ec__;
}
Expand Down Expand Up @@ -391,8 +391,8 @@ int geo_client::set_geo_data(const std::string &hash_key,
if (ec_ != PERR_OK) {
ret = ec_;
LOG_ERROR("set geo data failed. hash_key={}, sort_key={}, error={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
}
set_completed.notify();
Expand Down Expand Up @@ -521,8 +521,8 @@ void geo_client::async_search_radial(const std::string &hash_key,
](int ec_, std::string &&value_, pegasus_client::internal_info &&) mutable {
if (ec_ != PERR_OK) {
LOG_ERROR("get failed. hash_key={}, sort_key={}, error={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
get_error_string(ec_));
cb(ec_, {});
return;
Expand All @@ -531,8 +531,8 @@ void geo_client::async_search_radial(const std::string &hash_key,
S2LatLng latlng;
if (!_codec.decode_from_value(value_, latlng)) {
LOG_ERROR("decode_from_value failed. hash_key={}, sort_key={}, value={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
value_);
cb(ec_, {});
return;
Expand Down Expand Up @@ -726,8 +726,8 @@ bool geo_client::generate_geo_keys(const std::string &hash_key,
S2LatLng latlng;
if (!_codec.decode_from_value(value, latlng)) {
LOG_ERROR("decode_from_value failed. hash_key={}, sort_key={}, value={}",
hash_key,
sort_key,
utils::redact_sensitive_string(hash_key),
utils::redact_sensitive_string(sort_key),
value);
return false;
}
Expand Down Expand Up @@ -919,7 +919,8 @@ void geo_client::do_scan(pegasus_client::pegasus_scanner_wrapper scanner_wrapper
if (distance <= S2Earth::ToMeters(cap_ptr->radius())) {
std::string origin_hash_key, origin_sort_key;
if (!restore_origin_keys(geo_sort_key, origin_hash_key, origin_sort_key)) {
LOG_ERROR("restore_origin_keys failed. geo_sort_key={}", geo_sort_key);
LOG_ERROR("restore_origin_keys failed. geo_sort_key={}",
utils::redact_sensitive_string(geo_sort_key));
cb();
return;
}
Expand Down Expand Up @@ -955,10 +956,10 @@ int geo_client::distance(const std::string &hash_key1,
LOG_ERROR(
"get distance failed. hash_key1={}, sort_key1={}, hash_key2={}, sort_key2={}, "
"error={}",
hash_key1,
sort_key1,
hash_key2,
sort_key2,
utils::redact_sensitive_string(hash_key1),
utils::redact_sensitive_string(sort_key1),
utils::redact_sensitive_string(hash_key2),
utils::redact_sensitive_string(sort_key2),
get_error_string(ec_));
ret = ec_;
}
Expand Down Expand Up @@ -988,10 +989,10 @@ void geo_client::async_distance(const std::string &hash_key1,
if (ec_ != PERR_OK) {
LOG_ERROR("get data failed. hash_key1={}, sort_key1={}, hash_key2={}, sort_key2={}, "
"error={}",
hash_key1,
sort_key1,
hash_key2,
sort_key2,
utils::redact_sensitive_string(hash_key1),
utils::redact_sensitive_string(sort_key1),
utils::redact_sensitive_string(hash_key2),
utils::redact_sensitive_string(sort_key2),
get_error_string(ec_));
*ret = ec_;
}
Expand Down
6 changes: 3 additions & 3 deletions src/server/hotkey_collector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ inline void hotkey_collector::change_state_by_result()
if (!_result.hot_hash_key.empty()) {
change_state_to_finished();
LOG_ERROR_PREFIX("Find the hotkey: {}",
pegasus::utils::c_escape_string(_result.hot_hash_key));
pegasus::utils::c_escape_sensitive_string(_result.hot_hash_key));
}
break;
default:
Expand Down Expand Up @@ -277,7 +277,7 @@ void hotkey_collector::on_start_detect(dsn::replication::detect_hotkey_response
hint = fmt::format("{} hotkey result has been found: {}, you can send a stop rpc to "
"restart hotkey detection",
dsn::enum_to_string(_hotkey_type),
pegasus::utils::c_escape_string(_result.hot_hash_key));
pegasus::utils::c_escape_sensitive_string(_result.hot_hash_key));
break;
case hotkey_collector_state::STOPPED:
change_state_to_coarse_detecting();
Expand Down Expand Up @@ -314,7 +314,7 @@ void hotkey_collector::query_result(dsn::replication::detect_hotkey_response &re
LOG_INFO_PREFIX(hint);
} else {
resp.err = dsn::ERR_OK;
resp.__set_hotkey_result(pegasus::utils::c_escape_string(_result.hot_hash_key));
resp.__set_hotkey_result(pegasus::utils::c_escape_sensitive_string(_result.hot_hash_key));
}
}

Expand Down
Loading

0 comments on commit 13a45ab

Please sign in to comment.