Merge branch 'correct/time-millisecond-initialize' of github.com:Shyl…

…ock-Hg/nebula into correct/time-millisecond-initialize

.github/workflows/pull_request.yml

@@ -85,6 +85,7 @@ jobs:
                   -DENABLE_TESTING=on \
                   -B build
               echo "::set-output name=j::10"
+              echo "::set-output name=t::$(nproc)"
               ;;
             ubuntu2004)
               # build with Debug type
@@ -95,6 +96,7 @@ jobs:
                   -DENABLE_TESTING=on \
                   -B build
               echo "::set-output name=j::10"
+              echo "::set-output name=t::10"
               ;;
             esac
             ;;
@@ -108,12 +110,13 @@ jobs:
                 -DENABLE_TESTING=on \
                 -B build
             echo "::set-output name=j::6"
+            echo "::set-output name=t::10"
             ;;
           esac
       - name: Make
         run: |
           ccache -z
-          cmake --build build/ -j $(($(nproc)/2+1))
+          cmake --build build/ -j ${{ steps.cmake.outputs.t }}
           ccache -s
       - name: CTest
         env:

src/clients/meta/MetaClient.cpp

@@ -24,13 +24,17 @@
 #include "version/Version.h"
 #include "webservice/Common.h"
 
+DECLARE_int32(ws_meta_http_port);
+DECLARE_int32(ws_meta_h2_port);
+
 DEFINE_uint32(expired_time_factor, 5, "The factor of expired time based on heart beat interval");
 DEFINE_int32(heartbeat_interval_secs, 10, "Heartbeat interval in seconds");
 DEFINE_int32(meta_client_retry_times, 3, "meta client retry times, 0 means no retry");
 DEFINE_int32(meta_client_retry_interval_secs, 1, "meta client sleep interval between retry");
 DEFINE_int32(meta_client_timeout_ms, 60 * 1000, "meta client timeout");
 DEFINE_string(cluster_id_path, "cluster.id", "file path saved clusterId");
 DEFINE_int32(check_plan_killed_frequency, 8, "check plan killed every 1<<n times");
+
 namespace nebula {
 namespace meta {
 

src/clients/storage/GraphStorageClient.cpp

@@ -27,7 +27,7 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetNeighborsResponse>> GraphStorageCl
     bool random,
     const std::vector<cpp2::OrderBy>& orderBy,
     int64_t limit,
-    std::string filter,
+    const Expression* filter,
     folly::EventBase* evb) {
   auto cbStatus = getIdFromRow(space, false);
   if (!cbStatus.ok()) {
@@ -72,8 +72,8 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetNeighborsResponse>> GraphStorageCl
       spec.set_order_by(orderBy);
     }
     spec.set_limit(limit);
-    if (filter.size() > 0) {
-      spec.set_filter(filter);
+    if (filter != nullptr) {
+      spec.set_filter(filter->encode());
     }
     req.set_traverse_spec(std::move(spec));
   }
@@ -180,7 +180,7 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetPropResponse>> GraphStorageClient:
     bool dedup,
     const std::vector<cpp2::OrderBy>& orderBy,
     int64_t limit,
-    std::string filter,
+    const Expression* filter,
     folly::EventBase* evb) {
   auto cbStatus = getIdFromRow(space, edgeProps != nullptr);
   if (!cbStatus.ok()) {
@@ -216,8 +216,8 @@ folly::SemiFuture<StorageRpcResponse<cpp2::GetPropResponse>> GraphStorageClient:
       req.set_order_by(orderBy);
     }
     req.set_limit(limit);
-    if (filter.size() > 0) {
-      req.set_filter(filter);
+    if (filter != nullptr) {
+      req.set_filter(filter->encode());
     }
     req.set_common(common);
   }

src/clients/storage/GraphStorageClient.h

@@ -49,7 +49,7 @@ class GraphStorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsy
       bool random = false,
       const std::vector<cpp2::OrderBy>& orderBy = std::vector<cpp2::OrderBy>(),
       int64_t limit = std::numeric_limits<int64_t>::max(),
-      std::string filter = std::string(),
+      const Expression* filter = nullptr,
       folly::EventBase* evb = nullptr);
 
   folly::SemiFuture<StorageRpcResponse<cpp2::GetPropResponse>> getProps(
@@ -63,7 +63,7 @@ class GraphStorageClient : public StorageClientBase<cpp2::GraphStorageServiceAsy
       bool dedup = false,
       const std::vector<cpp2::OrderBy>& orderBy = std::vector<cpp2::OrderBy>(),
       int64_t limit = std::numeric_limits<int64_t>::max(),
-      std::string filter = std::string(),
+      const Expression* filter = nullptr,
       folly::EventBase* evb = nullptr);
 
   folly::SemiFuture<StorageRpcResponse<cpp2::ExecResponse>> addVertices(

src/common/algorithm/ReservoirSampling.h

@@ -37,7 +37,13 @@ class ReservoirSampling final {
     return false;
   }
 
-  std::vector<T>&& samples() && { return std::move(samples_); }
+  std::vector<T> samples() {
+    auto result = std::move(samples_);
+    samples_.clear();
+    samples_.reserve(num_);
+    cnt_ = 0;
+    return result;
+  }
 
  private:
   std::vector<T> samples_;

src/common/algorithm/test/ReservoirSamplingTest.cpp

@@ -18,7 +18,7 @@ TEST(ReservoirSamplingTest, Sample) {
       sampler.sampling(std::move(i));
     }
 
-    auto result = std::move(sampler).samples();
+    auto result = sampler.samples();
     EXPECT_EQ(5, result.size());
     for (auto i : result) {
       EXPECT_LE(0, i);
@@ -27,16 +27,18 @@ TEST(ReservoirSamplingTest, Sample) {
   }
   {
     ReservoirSampling<int64_t> sampler(5);
-    std::vector<int64_t> sampleSpace = {0, 1, 2};
-    for (auto i : sampleSpace) {
-      sampler.sampling(std::move(i));
-    }
+    for (size_t count = 0; count < 10; count++) {
+      std::vector<int64_t> sampleSpace = {0, 1, 2};
+      for (auto i : sampleSpace) {
+        sampler.sampling(std::move(i));
+      }
 
-    auto result = std::move(sampler).samples();
-    EXPECT_EQ(3, result.size());
-    EXPECT_EQ(0, result[0]);
-    EXPECT_EQ(1, result[1]);
-    EXPECT_EQ(2, result[2]);
+      auto result = sampler.samples();
+      EXPECT_EQ(3, result.size());
+      EXPECT_EQ(0, result[0]);
+      EXPECT_EQ(1, result[1]);
+      EXPECT_EQ(2, result[2]);
+    }
   }
 }
 }  // namespace algorithm

src/common/base/test/LoggingBenchmark.cpp

@@ -6,6 +6,9 @@
 
 #include <folly/Benchmark.h>
 #include <folly/init/Init.h>
+#include <folly/logging/FileHandlerFactory.h>
+#include <folly/logging/Init.h>
+#include <folly/logging/LoggerDB.h>
 #include <sys/types.h>
 
 #include <iostream>
@@ -18,6 +21,26 @@
               << "123";                 \
   }
 
+#define XLOG_SOMETHING(iters)           \
+  for (int64_t i = 0; i < iters; i++) { \
+    XLOG(INFO) << "Hello"               \
+               << " "                   \
+               << "Wolrd"               \
+               << "123";                \
+  }
+
+class XlogInit {
+ public:
+  explicit XlogInit(folly::StringPiece config) { folly::initLogging(config); }
+};
+
+static void xlogRegistFileHandler() {
+  folly::LoggerDB::get().registerHandlerFactory(std::make_unique<folly::FileHandlerFactory>());
+  // Since glog outputs the logs to /tmp by default, so we explicitly set a file handler for
+  // xlog and output logs to /tmp.
+  folly::initLogging(";default=file:path=/tmp/logging_bm.log");
+}
+
 /***************************
  *
  * <glog/logging.h> native
@@ -34,6 +57,9 @@ void loggingUsingGlog(int64_t iters) { LOG_SOMETHING(iters); }
 #include "common/base/Logging.h"
 void loggingOptimized(int64_t iters) { LOG_SOMETHING(iters); }
 
+#include <folly/logging/xlog.h>
+void loggingUsingXlog(int64_t iters) { XLOG_SOMETHING(iters); }
+
 /***************************
  *
  * Run benchmarks
@@ -49,6 +75,18 @@ BENCHMARK_RELATIVE(optimized_output_logs, iters) {
   loggingOptimized(iters);
 }
 
+BENCHMARK_RELATIVE(xlog_output_logs, iters) {
+  BENCHMARK_SUSPEND { static XlogInit init(".=INFO:default"); }
+  loggingUsingXlog(iters);
+}
+
+BENCHMARK_RELATIVE(xlog_output_logs_async, iters) {
+  BENCHMARK_SUSPEND {
+    static XlogInit init(".=INFO:default;default:async=true,max_buffer_size=4096");
+  }
+  loggingUsingXlog(iters);
+}
+
 BENCHMARK_DRAW_LINE();
 
 BENCHMARK(glog_skip_logs, iters) {
@@ -61,28 +99,35 @@ BENCHMARK_RELATIVE(optimized_skip_logs, iters) {
   loggingOptimized(iters);
 }
 
+BENCHMARK_RELATIVE(xlog_skip_logs, iters) {
+  BENCHMARK_SUSPEND { static XlogInit init(".=WARN:default"); }
+  loggingUsingXlog(iters);
+}
 /***************************
  *
  * main()
  *
  **************************/
 int main(int argc, char** argv) {
   folly::init(&argc, &argv, true);
+  xlogRegistFileHandler();
 
   folly::runBenchmarks();
   return 0;
 }
 
 /*
-Benchmark number is taken from WSL running on i7-8650
-
+Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz
 ============================================================================
-LoggingBenchmark.cpp                            relative  time/iter  iters/s
+src/common/base/test/LoggingBenchmark.cpp       relative  time/iter  iters/s
 ============================================================================
-glog_output_logs                                             3.13us  319.07K
-optimized_output_logs                            100.10%     3.13us  319.39K
+glog_output_logs                                             1.86us  536.82K
+optimized_output_logs                            100.26%     1.86us  538.24K
+xlog_output_logs                                  52.73%     3.53us  283.09K
+xlog_output_logs_async                            53.40%     3.49us  286.68K
 ----------------------------------------------------------------------------
-glog_skip_logs                                               1.76us  567.45K
-optimized_skip_logs                                 inf%     0.00fs  Infinity
+glog_skip_logs                                               1.27us  789.36K
+optimized_skip_logs                             94753.03%     1.34ns  747.94M
+xlog_skip_logs                                  5215.83%    24.29ns   41.17M
 ============================================================================
 */

src/common/datatypes/DataSet.h

@@ -7,6 +7,8 @@
 #ifndef COMMON_DATATYPES_DATASET_H_
 #define COMMON_DATATYPES_DATASET_H_
 
+#include <folly/dynamic.h>
+
 #include <iostream>
 #include <iterator>
 #include <sstream>
@@ -153,6 +155,44 @@ struct DataSet {
     return os.str();
   }
 
+  // format:
+  // [
+  //   {
+  //     "row": [ row-data ],
+  //     "meta": [ metadata ]
+  //   },
+  // ]
+  folly::dynamic toJson() const {
+    // parse rows to json
+    auto dataBody = folly::dynamic::array();
+    for (auto& row : rows) {
+      dataBody.push_back(rowToJson(row));
+    }
+
+    return dataBody;
+  }
+
+  // parse Nebula::Row to json
+  // format:
+  // {
+  //   "row": [ row-data ],
+  //   "meta": [ metadata ]
+  // }
+  folly::dynamic rowToJson(const Row& row) const {
+    folly::dynamic rowJsonObj = folly::dynamic::object();
+    auto rowDataList = folly::dynamic::array();
+    auto metaDataList = folly::dynamic::array();
+
+    for (const auto& ele : row.values) {
+      rowDataList.push_back(ele.toJson());
+      metaDataList.push_back(ele.getMetaData());
+    }
+
+    rowJsonObj.insert("row", rowDataList);
+    rowJsonObj.insert("meta", metaDataList);
+    return rowJsonObj;
+  }
+
   bool operator==(const DataSet& rhs) const { return colNames == rhs.colNames && rows == rhs.rows; }
 };
 

src/common/datatypes/Date.h

@@ -7,6 +7,8 @@
 #ifndef COMMON_DATATYPES_DATE_H_
 #define COMMON_DATATYPES_DATE_H_
 
+#include <folly/dynamic.h>
+
 #include <string>
 
 namespace nebula {
@@ -62,6 +64,7 @@ struct Date {
   Date operator-(int64_t days) const;
 
   std::string toString() const;
+  folly::dynamic toJson() const { return toString(); }
 
   // Return the number of days since -32768/1/1
   int64_t toInt() const;
@@ -113,6 +116,8 @@ struct Time {
   }
 
   std::string toString() const;
+  // 'Z' representing UTC timezone
+  folly::dynamic toJson() const { return toString() + "Z"; }
 };
 
 inline std::ostream& operator<<(std::ostream& os, const Time& d) {
@@ -203,6 +208,8 @@ struct DateTime {
   }
 
   std::string toString() const;
+  // 'Z' representing UTC timezone
+  folly::dynamic toJson() const { return toString() + "Z"; }
 };
 
 inline std::ostream& operator<<(std::ostream& os, const DateTime& d) {