feat(encryption): add kms key management

src/replica/CMakeLists.txt

@@ -58,6 +58,7 @@ set(MY_PROJ_SRC
 set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS
+        absl::strings
         dsn_replication_common
         dsn.failure_detector
         dsn.block_service
@@ -66,9 +67,11 @@ set(MY_PROJ_LIBS
         dsn_nfs
         dsn_dist_cmd
         dsn_http
+        curl
         dsn_runtime
         dsn_aio
         dsn_meta_server
+        dsn.security
         rocksdb
         lz4
         zstd

src/replica/default_key_provider.h

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <openssl/rand.h>
+
+#include "absl/strings/escaping.h"
+#include "replica/key_provider.h"
+#include "utils/error_code.h"
+#include "utils/fmt_logging.h"
+
+namespace dsn {
+namespace security {
+
+class DefaultKeyProvider : public KeyProvider
+{
+public:
+    ~DefaultKeyProvider() override {}
+    dsn::error_s DecryptEncryptionKey(const std::string &encryption_key,
+                                      const std::string & /*iv*/,
+                                      const std::string & /*key_version*/,
+                                      std::string *decrypted_key) override
+    {
+        *decrypted_key = ::absl::HexStringToBytes(encryption_key);
+
+#ifdef __linux__
+        memfrob(decrypted_key->data(), decrypted_key->length());
+#else
+        // On Linux, memfrob() bitwise XORs the data with the magic number that is
+        // the answer to the ultimate question of life, the universe, and
+        // everything. On Mac, we do this manually.
+        const uint8_t kMagic = 42;
+        for (auto i = 0; i < decrypted_key->length(); ++i) {
+            decrypted_key->data()[i] ^= kMagic;
+        }
+#endif
+        *decrypted_key = ::absl::BytesToHexString(*decrypted_key);
+        return dsn::error_s::ok();
+    }
+
+    dsn::error_s GenerateEncryptionKey(std::string *encryption_key,
+                                       std::string *iv,
+                                       std::string *key_version) override
+    {
+        unsigned char key_bytes[32];
+        unsigned char iv_bytes[32];
+        int num_bytes = 16;
+        RAND_bytes(key_bytes, num_bytes);
+        std::string tmp = reinterpret_cast<const char *>(key_bytes);
+        std::string *decrypted_key = &tmp;
+#ifdef __linux__
+        memfrob(decrypted_key->data(), decrypted_key->length());
+#else
+        // On Linux, memfrob() bitwise XORs the data with the magic number that is
+        // the answer to the ultimate question of life, the universe, and
+        // everything. On Mac, we do this manually.
+        const uint8_t kMagic = 42;
+        for (auto i = 0; i < decrypted_key->length(); ++i) {
+            decrypted_key->data()[i] ^= kMagic;
+        }
+#endif
+        std::string dek = *decrypted_key;
+        *encryption_key = ::absl::BytesToHexString(dek);
+        RAND_bytes(iv_bytes, num_bytes);
+        *iv = ::absl::BytesToHexString(reinterpret_cast<const char *>(iv_bytes));
+        *key_version = "encryptionkey@0";
+        return dsn::error_s::ok();
+    }
+};
+} // namespace security
+} // namespace dsn

src/replica/key_provider.h

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+
+#include "utils/errors.h"
+
+namespace dsn {
+namespace security {
+
+// An interface for encrypting and decrypting Pegasus's encryption keys.
+class KeyProvider
+{
+public:
+    virtual ~KeyProvider() = default;
+
+    // Decrypts the encryption key.
+    virtual dsn::error_s DecryptEncryptionKey(const std::string &encryption_key,
+                                              const std::string &iv,
+                                              const std::string &key_version,
+                                              std::string *decrypted_key) = 0;
+
+    // Generates an encryption key (the generated key is encrypted).
+    virtual dsn::error_s GenerateEncryptionKey(std::string *encryption_key,
+                                               std::string *iv,
+                                               std::string *key_version) = 0;
+};
+} // namespace security
+} // namespace dsn

src/replica/pegasus_kms_key_provider.cpp

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <string>
+
+#include "replica/pegasus_kms_key_provider.h"
+#include "utils/errors.h"
+
+namespace dsn {
+namespace security {
+
+dsn::error_s PegasusKMSKeyProvider::DecryptEncryptionKey(const std::string &encryption_key,
+                                                         const std::string &iv,
+                                                         const std::string &key_version,
+                                                         std::string *decrypted_key)
+{
+    return client_.DecryptEncryptionKey(encryption_key, iv, key_version, decrypted_key);
+}
+
+dsn::error_s PegasusKMSKeyProvider::GenerateEncryptionKey(std::string *encryption_key,
+                                                          std::string *iv,
+                                                          std::string *key_version)
+{
+    return client_.GenerateEncryptionKey(encryption_key, iv, key_version);
+}
+
+} // namespace security
+} // namespace dsn

src/replica/pegasus_kms_key_provider.h

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <utility>
+
+#include "replica/key_provider.h"
+#include "runtime/security/kms_client.h"
+#include "utils/errors.h"
+
+namespace dsn {
+namespace security {
+class PegasusKMSKeyProvider : public KeyProvider
+{
+public:
+    ~PegasusKMSKeyProvider() override {}
+
+    PegasusKMSKeyProvider(const std::vector<std::string> &kms_url, std::string cluster_key_name)
+        : client_(kms_url, std::move(cluster_key_name))
+    {
+    }
+
+    // Decrypts the encryption key.
+    dsn::error_s DecryptEncryptionKey(const std::string &encryption_key,
+                                      const std::string &iv,
+                                      const std::string &key_version,
+                                      std::string *decrypted_key) override;
+
+    // Generates an encryption key (the generated key is encrypted).
+    dsn::error_s GenerateEncryptionKey(std::string *encryption_key,
+                                       std::string *iv,
+                                       std::string *key_version) override;
+
+private:
+    KMSClient client_;
+};
+} // namespace security
+} // namespace dsn

src/replica/replica_stub.cpp

@@ -56,19 +56,23 @@
 #include "nfs_types.h"
 #include "replica.h"
 #include "replica/duplication/replica_follower.h"
+#include "replica/key_provider.h"
+#include "replica/log_file.h"
+#include "replica/pegasus_kms_key_provider.h"
 #include "replica/replica_context.h"
 #include "replica/replica_stub.h"
 #include "replica/replication_app_base.h"
 #include "replica_disk_migrator.h"
-#include "replica_stub.h"
 #include "runtime/api_layer1.h"
 #include "runtime/ranger/access_type.h"
 #include "runtime/rpc/rpc_message.h"
 #include "runtime/rpc/serialization.h"
 #include "runtime/security/access_controller.h"
+#include "runtime/security/replica_kms_info.h"
 #include "runtime/task/async_calls.h"
 #include "split/replica_split_manager.h"
 #include "utils/command_manager.h"
+#include "utils/errors.h"
 #include "utils/filesystem.h"
 #include "utils/fmt_logging.h"
 #include "utils/ports.h"
@@ -214,7 +218,14 @@ METRIC_DEFINE_gauge_int64(server,
                           dsn::metric_unit::kBytes,
                           "The max size of copied files among all splitting replicas");
 
+DSN_DECLARE_bool(encrypt_data_at_rest);
+DSN_DECLARE_string(server_key);
+
 namespace dsn {
+
+namespace security {
+DSN_DECLARE_bool(enable_acl);
+}
 namespace replication {
 DSN_DEFINE_bool(replication,
                 deny_client_on_start,
@@ -291,6 +302,15 @@ DSN_DEFINE_int32(
     10,
     "if tcmalloc reserved but not-used memory exceed this percentage of application allocated "
     "memory, replica server will release the exceeding memory back to operating system");
+DSN_DEFINE_string(pegasus.server,
+                  encryption_cluster_key_name,
+                  "pegasus",
+                  "The cluster name of encrypted server which use to get server key from kms.");
+
+DSN_DEFINE_string(pegasus.server,
+                  hadoop_kms_url,
+                  "",
+                  "Where the server key of file system can get from.");
 
 DSN_DECLARE_bool(duplication_enabled);
 DSN_DECLARE_int32(fd_beacon_interval_seconds);
@@ -350,6 +370,11 @@ replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/,
     _failure_detector = nullptr;
     _state = NS_Disconnected;
     _primary_address_str[0] = '\0';
+    if (FLAGS_encrypt_data_at_rest) {
+        key_provider.reset(new dsn::security::PegasusKMSKeyProvider(
+            ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()),
+            FLAGS_encryption_cluster_key_name));
+    }
 }
 
 replica_stub::~replica_stub(void) { close(); }
@@ -362,6 +387,35 @@ void replica_stub::initialize(bool clear /* = false*/)
     _access_controller = std::make_unique<dsn::security::access_controller>();
 }
 
+dsn::error_s store_kms_key(const std::string &data_dir,
+                           const std::string &encryption_key,
+                           const std::string &iv,
+                           const std::string &key_version)
+{
+    replica_kms_info kms_info(encryption_key, iv, key_version);
+    auto err = kms_info.store(data_dir);
+    if (err != dsn::ERR_OK) {
+        return dsn::error_s::make(err, "Can't open kms-info file to write");
+    }
+    return dsn::error_s::ok();
+}
+
+void get_kms_key(const std::string &data_dir,
+                 std::string *encryption_key,
+                 std::string *iv,
+                 std::string *key_version)
+{
+    replica_kms_info kms_info;
+    auto err = kms_info.load(data_dir);
+    *encryption_key = kms_info.encryption_key;
+    *iv = kms_info.iv;
+    *key_version = kms_info.key_version;
+    if (err != dsn::ERR_OK) {
+        LOG_WARNING("Can't open kms-info file to read, this is normal when first launch process");
+    }
+    return;
+}
+
 void replica_stub::initialize(const replication_options &opts, bool clear /* = false*/)
 {
     _primary_address = dsn_primary_address();
@@ -389,9 +443,36 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f
         }
     }
 
+    std::string encryption_key;
+    std::string iv;
+    std::string key_version;
+    std::string server_key;
+    // get and store Encrypted Encryption Key(eek),Initialization Vector(iv),Key Version from kms
+    if (key_provider && !utils::is_empty(FLAGS_hadoop_kms_url)) {
+        get_kms_key(_options.data_dirs[0], &encryption_key, &iv, &key_version);
+        // The encryption key should empty when process upon the first launch. And the process will
+        // get eek,iv,kv from kms
+        // After first launch, the encryption key should not empty and get from kms-info file. The
+        // process get Decrypted Encryption Key(dek) from kms
+        if (encryption_key.empty()) {
+            auto err = key_provider->GenerateEncryptionKey(&encryption_key, &iv, &key_version);
+            CHECK(key_provider->GenerateEncryptionKey(&encryption_key, &iv, &key_version),
+                  "get encryption key failed, err = {}",
+                  err);
+        }
+        CHECK(key_provider->DecryptEncryptionKey(encryption_key, iv, key_version, &server_key),
+              "get decryption key failed");
+        FLAGS_server_key = server_key.c_str();
+    }
+
     // Initialize the file system manager.
     _fs_manager.initialize(_options.data_dirs, _options.data_dir_tags);
 
+    if (key_provider && !utils::is_empty(FLAGS_hadoop_kms_url)) {
+        CHECK(store_kms_key(_options.data_dirs[0], encryption_key, iv, key_version),
+              "Cant store kms key");
+    }
+
     // Check slog is not exist.
     auto full_slog_path = fmt::format("{}/replica/slog/", _options.slog_dir);
     if (utils::filesystem::directory_exists(full_slog_path)) {