Support TLS transport encryption (#2584)

* Add the ssl support. * Add tests to tck. * Support CA signed certificate. * Remove the pssword configuration. * Remove the comment. * Support independent meta server ssl. * Initialize the ssl when enable meta ssl. * Fix typo. * Fix the header order. * clear logic. * Add test for ca signed mode. * Fix flag note.
vesoft-inc · Sep 24, 2021 · 6fa47a4 · 6fa47a4
1 parent 3941945
commit 6fa47a4
Show file tree

Hide file tree

Showing 44 changed files with 346 additions and 16 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -127,7 +127,24 @@ jobs:
         timeout-minutes: 20
       - name: Setup cluster
         run: |
-          make up
+          case ${{ matrix.compiler }} in
+          gcc-*)
+            case ${{ matrix.os }} in
+            centos7)
+              # normal cluster
+              make up
+              ;;
+            ubuntu2004)
+              # ssl cluster
+              make ENABLE_SSL=true CA_SIGNED=true up
+              ;;
+            esac
+            ;;
+          clang-*)
+            # graph ssl only cluster
+            make ENABLE_SSL=false ENABLE_GRAPH_SSL=true up
+            ;;
+          esac
         working-directory: tests/
         timeout-minutes: 2
       - name: Pytest

diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp
@@ -19,6 +19,7 @@
 #include "common/http/HttpClient.h"
 #include "common/meta/NebulaSchemaProvider.h"
 #include "common/network/NetworkUtils.h"
+#include "common/ssl/SSLConfig.h"
 #include "common/stats/StatsManager.h"
 #include "common/time/TimeUtils.h"
 #include "version/Version.h"
@@ -49,7 +50,8 @@ MetaClient::MetaClient(std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool
   CHECK(ioThreadPool_ != nullptr) << "IOThreadPool is required";
   CHECK(!addrs_.empty())
       << "No meta server address is specified or can be solved. Meta server is required";
-  clientsMan_ = std::make_shared<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>>();
+  clientsMan_ = std::make_shared<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>>(
+      FLAGS_enable_ssl || FLAGS_enable_meta_ssl);
   updateActive();
   updateLeader();
   bgThread_ = std::make_unique<thread::GenericWorker>();

diff --git a/src/clients/meta/test/CMakeLists.txt b/src/clients/meta/test/CMakeLists.txt
@@ -10,5 +10,6 @@ nebula_add_test(
         $<TARGET_OBJECTS:file_based_cluster_id_man_obj>
         $<TARGET_OBJECTS:base_obj>
         $<TARGET_OBJECTS:fs_obj>
+        $<TARGET_OBJECTS:ssl_obj>
     LIBRARIES gtest
 )
diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h
@@ -8,6 +8,7 @@
 
 #include <folly/Try.h>
 
+#include "common/ssl/SSLConfig.h"
 #include "common/time/WallClock.h"
 
 namespace nebula {
@@ -72,7 +73,7 @@ template <typename ClientType>
 StorageClientBase<ClientType>::StorageClientBase(
     std::shared_ptr<folly::IOThreadPoolExecutor> threadPool, meta::MetaClient* metaClient)
     : metaClient_(metaClient), ioThreadPool_(threadPool) {
-  clientsMan_ = std::make_unique<thrift::ThriftClientManager<ClientType>>();
+  clientsMan_ = std::make_unique<thrift::ThriftClientManager<ClientType>>(FLAGS_enable_ssl);
 }
 
 template <typename ClientType>

diff --git a/src/codec/test/CMakeLists.txt b/src/codec/test/CMakeLists.txt
@@ -31,6 +31,7 @@ set(CODEC_TEST_LIBS
     $<TARGET_OBJECTS:agg_function_manager_obj>
     $<TARGET_OBJECTS:time_utils_obj>
     $<TARGET_OBJECTS:version_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 

diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
@@ -25,3 +25,4 @@ nebula_add_subdirectory(function)
 nebula_add_subdirectory(graph)
 nebula_add_subdirectory(plugin)
 nebula_add_subdirectory(utils)
+nebula_add_subdirectory(ssl)
diff --git a/src/common/ssl/CMakeLists.txt b/src/common/ssl/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright (c) 2021 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License,
+# attached with Common Clause Condition 1.0, found in the LICENSES directory.
+
+nebula_add_library(
+    ssl_obj
+    OBJECT
+    SSLConfig.cpp
+)
diff --git a/src/common/ssl/SSLConfig.cpp b/src/common/ssl/SSLConfig.cpp
@@ -0,0 +1,38 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include "common/ssl/SSLConfig.h"
+
+DEFINE_string(cert_path, "", "Path to cert pem.");
+DEFINE_string(key_path, "", "Path to cert key.");
+DEFINE_string(ca_path, "", "Path to trusted CA file.");
+DEFINE_bool(enable_ssl, false, "Whether to enable ssl.");
+DEFINE_bool(enable_graph_ssl, false, "Whether to enable ssl of graph server.");
+DEFINE_bool(enable_meta_ssl, false, "Whether to enable ssl of meta server.");
+
+namespace nebula {
+
+std::shared_ptr<wangle::SSLContextConfig> sslContextConfig() {
+  auto sslCfg = std::make_shared<wangle::SSLContextConfig>();
+  sslCfg->addCertificate(FLAGS_cert_path, FLAGS_key_path, "");
+  sslCfg->isDefault = true;
+  return sslCfg;
+}
+
+std::shared_ptr<folly::SSLContext> createSSLContext() {
+  auto context = std::make_shared<folly::SSLContext>();
+  if (!FLAGS_ca_path.empty()) {
+    context->loadTrustedCertificates(FLAGS_ca_path.c_str());
+    // don't do peer name validation
+    context->authenticate(true, false);
+    // verify the server cert
+    context->setVerificationOption(folly::SSLContext::SSLVerifyPeerEnum::VERIFY);
+  }
+  folly::ssl::setSignatureAlgorithms<folly::ssl::SSLCommonOptions>(*context);
+  return context;
+}
+
+}  // namespace nebula
diff --git a/src/common/ssl/SSLConfig.h b/src/common/ssl/SSLConfig.h
@@ -0,0 +1,26 @@
+
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#pragma once
+
+#include <folly/io/async/SSLContext.h>
+#include <gflags/gflags.h>
+#include <wangle/ssl/SSLContextConfig.h>
+
+#include <memory>
+
+DECLARE_bool(enable_ssl);
+DECLARE_bool(enable_graph_ssl);
+DECLARE_bool(enable_meta_ssl);
+
+namespace nebula {
+
+extern std::shared_ptr<wangle::SSLContextConfig> sslContextConfig();
+
+extern std::shared_ptr<folly::SSLContext> createSSLContext();
+
+}  // namespace nebula
diff --git a/src/common/thrift/ThriftClientManager-inl.h b/src/common/thrift/ThriftClientManager-inl.h
@@ -6,11 +6,13 @@
 
 #pragma once
 
+#include <folly/io/async/AsyncSSLSocket.h>
 #include <folly/io/async/AsyncSocket.h>
 #include <folly/system/ThreadName.h>
 #include <thrift/lib/cpp2/async/HeaderClientChannel.h>
 
 #include "common/network/NetworkUtils.h"
+#include "common/ssl/SSLConfig.h"
 
 DECLARE_int32(conn_timeout_ms);
 
@@ -71,9 +73,14 @@ std::shared_ptr<ClientType> ThriftClientManager<ClientType>::client(const HostAd
 
   VLOG(2) << "Connecting to " << host << " for " << ++connectionCount << " times";
   std::shared_ptr<folly::AsyncSocket> socket;
-  evb->runImmediatelyOrRunInEventBaseThreadAndWait([&socket, evb, resolved]() {
-    socket =
-        folly::AsyncSocket::newSocket(evb, resolved.host, resolved.port, FLAGS_conn_timeout_ms);
+  evb->runImmediatelyOrRunInEventBaseThreadAndWait([this, &socket, evb, resolved]() {
+    if (enableSSL_) {
+      socket = folly::AsyncSSLSocket::newSocket(nebula::createSSLContext(), evb);
+      socket->connect(nullptr, resolved.host, resolved.port, FLAGS_conn_timeout_ms);
+    } else {
+      socket =
+          folly::AsyncSocket::newSocket(evb, resolved.host, resolved.port, FLAGS_conn_timeout_ms);
+    }
   });
   auto headerClientChannel = apache::thrift::HeaderClientChannel::newChannel(socket);
   if (timeout > 0) {

diff --git a/src/common/thrift/ThriftClientManager.h b/src/common/thrift/ThriftClientManager.h
@@ -7,6 +7,7 @@
 #ifndef COMMON_THRIFT_THRIFTCLIENTMANAGER_H_
 #define COMMON_THRIFT_THRIFTCLIENTMANAGER_H_
 
+#include <folly/io/async/AsyncSocket.h>
 #include <folly/io/async/EventBaseManager.h>
 
 #include "common/base/Base.h"
@@ -25,7 +26,9 @@ class ThriftClientManager final {
 
   ~ThriftClientManager() { VLOG(3) << "~ThriftClientManager"; }
 
-  ThriftClientManager() { VLOG(3) << "ThriftClientManager"; }
+  explicit ThriftClientManager(bool enableSSL = false) : enableSSL_(enableSSL) {
+    VLOG(3) << "ThriftClientManager";
+  }
 
  private:
   using ClientMap = std::unordered_map<std::pair<HostAddr, folly::EventBase*>,  // <ip, port>
@@ -34,6 +37,8 @@ class ThriftClientManager final {
                                        >;
 
   folly::ThreadLocal<ClientMap> clientMap_;
+  // whether enable ssl
+  bool enableSSL_{false};
 };
 
 }  // namespace thrift

diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt
@@ -29,6 +29,7 @@ set(common_deps
     $<TARGET_OBJECTS:agg_function_manager_obj>
     $<TARGET_OBJECTS:time_utils_obj>
     $<TARGET_OBJECTS:version_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 set(storage_meta_deps

diff --git a/src/daemons/GraphDaemon.cpp b/src/daemons/GraphDaemon.cpp
@@ -5,6 +5,7 @@
  */
 
 #include <errno.h>
+#include <folly/ssl/Init.h>
 #include <signal.h>
 #include <string.h>
 #include <thrift/lib/cpp2/server/ThriftServer.h>
@@ -15,6 +16,7 @@
 #include "common/fs/FileUtils.h"
 #include "common/network/NetworkUtils.h"
 #include "common/process/ProcessUtils.h"
+#include "common/ssl/SSLConfig.h"
 #include "common/time/TimezoneInfo.h"
 #include "graph/service/GraphFlags.h"
 #include "graph/service/GraphService.h"
@@ -52,6 +54,9 @@ int main(int argc, char *argv[]) {
   }
 
   folly::init(&argc, &argv, true);
+  if (FLAGS_enable_ssl || FLAGS_enable_graph_ssl || FLAGS_enable_meta_ssl) {
+    folly::ssl::init();
+  }
   nebula::initCounters();
 
   if (FLAGS_flagfile.empty()) {
@@ -149,6 +154,9 @@ int main(int argc, char *argv[]) {
   gServer->setIdleTimeout(std::chrono::seconds(FLAGS_client_idle_timeout_secs));
   gServer->setNumAcceptThreads(FLAGS_num_accept_threads);
   gServer->setListenBacklog(FLAGS_listen_backlog);
+  if (FLAGS_enable_ssl || FLAGS_enable_graph_ssl) {
+    gServer->setSSLConfig(nebula::sslContextConfig());
+  }
   setupThreadManager();
 
   // Setup the signal handlers

diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp
@@ -4,6 +4,7 @@
  * attached with Common Clause Condition 1.0, found in the LICENSES directory.
  */
 
+#include <folly/ssl/Init.h>
 #include <thrift/lib/cpp2/server/ThriftServer.h>
 
 #include "common/base/Base.h"
@@ -12,6 +13,7 @@
 #include "common/hdfs/HdfsHelper.h"
 #include "common/network/NetworkUtils.h"
 #include "common/process/ProcessUtils.h"
+#include "common/ssl/SSLConfig.h"
 #include "common/thread/GenericThreadPool.h"
 #include "common/time/TimezoneInfo.h"
 #include "kvstore/NebulaStore.h"
@@ -204,6 +206,9 @@ int main(int argc, char* argv[]) {
   }
 
   folly::init(&argc, &argv, true);
+  if (FLAGS_enable_ssl || FLAGS_enable_meta_ssl) {
+    folly::ssl::init();
+  }
   if (FLAGS_data_path.empty()) {
     LOG(ERROR) << "Meta Data Path should not empty";
     return EXIT_FAILURE;
@@ -307,6 +312,9 @@ int main(int argc, char* argv[]) {
     gServer->setPort(FLAGS_port);
     gServer->setIdleTimeout(std::chrono::seconds(0));  // No idle timeout on client connection
     gServer->setInterface(std::move(handler));
+    if (FLAGS_enable_ssl || FLAGS_enable_meta_ssl) {
+      gServer->setSSLConfig(nebula::sslContextConfig());
+    }
     gServer->serve();  // Will wait until the server shuts down
   } catch (const std::exception& e) {
     LOG(ERROR) << "Exception thrown: " << e.what();

diff --git a/src/daemons/StorageDaemon.cpp b/src/daemons/StorageDaemon.cpp
@@ -4,6 +4,7 @@
  * attached with Common Clause Condition 1.0, found in the LICENSES directory.
  */
 
+#include <folly/ssl/Init.h>
 #include <thrift/lib/cpp2/server/ThriftServer.h>
 
 #include "common/base/Base.h"
@@ -69,6 +70,9 @@ int main(int argc, char *argv[]) {
   }
 
   folly::init(&argc, &argv, true);
+  if (FLAGS_enable_ssl || FLAGS_enable_meta_ssl) {
+    folly::ssl::init();
+  }
   if (FLAGS_daemonize) {
     google::SetStderrLogging(google::FATAL);
   } else {

diff --git a/src/graph/context/test/CMakeLists.txt b/src/graph/context/test/CMakeLists.txt
@@ -40,6 +40,7 @@ SET(CONTEXT_TEST_LIBS
     $<TARGET_OBJECTS:graph_session_obj>
     $<TARGET_OBJECTS:planner_obj>
     $<TARGET_OBJECTS:idgenerator_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 nebula_add_test(

diff --git a/src/graph/executor/test/CMakeLists.txt b/src/graph/executor/test/CMakeLists.txt
@@ -47,6 +47,7 @@ SET(EXEC_QUERY_TEST_OBJS
     $<TARGET_OBJECTS:graph_auth_obj>
     $<TARGET_OBJECTS:expr_visitor_obj>
     $<TARGET_OBJECTS:graph_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 SET(EXEC_QUERY_TEST_LIBS

diff --git a/src/graph/optimizer/test/CMakeLists.txt b/src/graph/optimizer/test/CMakeLists.txt
@@ -43,6 +43,7 @@ set(OPTIMIZER_TEST_LIB
     $<TARGET_OBJECTS:graph_context_obj>
     $<TARGET_OBJECTS:validator_obj>
     $<TARGET_OBJECTS:optimizer_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 nebula_add_test(

diff --git a/src/graph/util/test/CMakeLists.txt b/src/graph/util/test/CMakeLists.txt
@@ -32,6 +32,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
         $<TARGET_OBJECTS:ws_common_obj>
         $<TARGET_OBJECTS:version_obj>
+        $<TARGET_OBJECTS:ssl_obj>
         $<TARGET_OBJECTS:idgenerator_obj>
         $<TARGET_OBJECTS:expr_visitor_obj>
         $<TARGET_OBJECTS:graph_session_obj>

diff --git a/src/graph/validator/test/CMakeLists.txt b/src/graph/validator/test/CMakeLists.txt
@@ -51,6 +51,7 @@ set(VALIDATOR_TEST_LIBS
     $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
     $<TARGET_OBJECTS:ws_common_obj>
     $<TARGET_OBJECTS:version_obj>
+    $<TARGET_OBJECTS:ssl_obj>
 )
 
 nebula_add_test(

diff --git a/src/graph/visitor/test/CMakeLists.txt b/src/graph/visitor/test/CMakeLists.txt
@@ -56,6 +56,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
         $<TARGET_OBJECTS:ws_common_obj>
         $<TARGET_OBJECTS:version_obj>
+        $<TARGET_OBJECTS:ssl_obj>
     LIBRARIES
         gtest
         ${THRIFT_LIBRARIES}

diff --git a/src/kvstore/NebulaStore.h b/src/kvstore/NebulaStore.h
@@ -11,6 +11,7 @@
 #include <gtest/gtest_prod.h>
 
 #include "common/base/Base.h"
+#include "common/ssl/SSLConfig.h"
 #include "common/utils/Utils.h"
 #include "interface/gen-cpp2/RaftexServiceAsyncClient.h"
 #include "kvstore/DiskManager.h"
@@ -65,7 +66,8 @@ class NebulaStore : public KVStore, public Handler {
         options_(std::move(options)) {
     CHECK_NOTNULL(options_.partMan_);
     clientMan_ =
-        std::make_shared<thrift::ThriftClientManager<raftex::cpp2::RaftexServiceAsyncClient>>();
+        std::make_shared<thrift::ThriftClientManager<raftex::cpp2::RaftexServiceAsyncClient>>(
+            FLAGS_enable_ssl);
   }
 
   ~NebulaStore();

diff --git a/src/kvstore/raftex/RaftexService.cpp b/src/kvstore/raftex/RaftexService.cpp
@@ -9,6 +9,7 @@
 #include <folly/ScopeGuard.h>
 
 #include "common/base/Base.h"
+#include "common/ssl/SSLConfig.h"
 #include "kvstore/raftex/RaftPart.h"
 
 namespace nebula {
@@ -60,6 +61,9 @@ void RaftexService::initThriftServer(std::shared_ptr<folly::IOThreadPoolExecutor
                                      std::shared_ptr<folly::Executor> workers,
                                      uint16_t port) {
   LOG(INFO) << "Init thrift server for raft service, port: " << port;
+  if (FLAGS_enable_ssl) {
+    server_->setSSLConfig(nebula::sslContextConfig());
+  }
   server_->setPort(port);
   server_->setIdleTimeout(std::chrono::seconds(0));
   if (pool != nullptr) {