diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d6e701e65f7..54ce3508195 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -85,6 +85,7 @@ jobs: -DENABLE_TESTING=on \ -B build echo "::set-output name=j::10" + echo "::set-output name=t::$(nproc)" ;; ubuntu2004) # build with Debug type @@ -95,6 +96,7 @@ jobs: -DENABLE_TESTING=on \ -B build echo "::set-output name=j::10" + echo "::set-output name=t::10" ;; esac ;; @@ -108,12 +110,13 @@ jobs: -DENABLE_TESTING=on \ -B build echo "::set-output name=j::6" + echo "::set-output name=t::10" ;; esac - name: Make run: | ccache -z - cmake --build build/ -j $(($(nproc)/2+1)) + cmake --build build/ -j ${{ steps.cmake.outputs.t }} ccache -s - name: CTest env: diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 4fb255962e0..87e276e5409 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -24,6 +24,9 @@ #include "version/Version.h" #include "webservice/Common.h" +DECLARE_int32(ws_meta_http_port); +DECLARE_int32(ws_meta_h2_port); + DEFINE_uint32(expired_time_factor, 5, "The factor of expired time based on heart beat interval"); DEFINE_int32(heartbeat_interval_secs, 10, "Heartbeat interval in seconds"); DEFINE_int32(meta_client_retry_times, 3, "meta client retry times, 0 means no retry"); @@ -31,6 +34,7 @@ DEFINE_int32(meta_client_retry_interval_secs, 1, "meta client sleep interval bet DEFINE_int32(meta_client_timeout_ms, 60 * 1000, "meta client timeout"); DEFINE_string(cluster_id_path, "cluster.id", "file path saved clusterId"); DEFINE_int32(check_plan_killed_frequency, 8, "check plan killed every 1<> GraphStorageCl bool random, const std::vector& orderBy, int64_t limit, - std::string filter, + const Expression* filter, folly::EventBase* evb) { auto cbStatus = getIdFromRow(space, false); if (!cbStatus.ok()) { @@ -72,8 +72,8 @@ folly::SemiFuture> GraphStorageCl spec.set_order_by(orderBy); } spec.set_limit(limit); - if (filter.size() > 0) { - spec.set_filter(filter); + if (filter != nullptr) { + spec.set_filter(filter->encode()); } req.set_traverse_spec(std::move(spec)); } @@ -180,7 +180,7 @@ folly::SemiFuture> GraphStorageClient: bool dedup, const std::vector& orderBy, int64_t limit, - std::string filter, + const Expression* filter, folly::EventBase* evb) { auto cbStatus = getIdFromRow(space, edgeProps != nullptr); if (!cbStatus.ok()) { @@ -216,8 +216,8 @@ folly::SemiFuture> GraphStorageClient: req.set_order_by(orderBy); } req.set_limit(limit); - if (filter.size() > 0) { - req.set_filter(filter); + if (filter != nullptr) { + req.set_filter(filter->encode()); } req.set_common(common); } diff --git a/src/clients/storage/GraphStorageClient.h b/src/clients/storage/GraphStorageClient.h index 665234fbda6..2cbddab3754 100644 --- a/src/clients/storage/GraphStorageClient.h +++ b/src/clients/storage/GraphStorageClient.h @@ -49,7 +49,7 @@ class GraphStorageClient : public StorageClientBase& orderBy = std::vector(), int64_t limit = std::numeric_limits::max(), - std::string filter = std::string(), + const Expression* filter = nullptr, folly::EventBase* evb = nullptr); folly::SemiFuture> getProps( @@ -63,7 +63,7 @@ class GraphStorageClient : public StorageClientBase& orderBy = std::vector(), int64_t limit = std::numeric_limits::max(), - std::string filter = std::string(), + const Expression* filter = nullptr, folly::EventBase* evb = nullptr); folly::SemiFuture> addVertices( diff --git a/src/common/algorithm/ReservoirSampling.h b/src/common/algorithm/ReservoirSampling.h index 6d846e1ad10..04cc937ad66 100644 --- a/src/common/algorithm/ReservoirSampling.h +++ b/src/common/algorithm/ReservoirSampling.h @@ -37,7 +37,13 @@ class ReservoirSampling final { return false; } - std::vector&& samples() && { return std::move(samples_); } + std::vector samples() { + auto result = std::move(samples_); + samples_.clear(); + samples_.reserve(num_); + cnt_ = 0; + return result; + } private: std::vector samples_; diff --git a/src/common/algorithm/test/ReservoirSamplingTest.cpp b/src/common/algorithm/test/ReservoirSamplingTest.cpp index 67e2c22e8ac..1a5d6c32bbb 100644 --- a/src/common/algorithm/test/ReservoirSamplingTest.cpp +++ b/src/common/algorithm/test/ReservoirSamplingTest.cpp @@ -18,7 +18,7 @@ TEST(ReservoirSamplingTest, Sample) { sampler.sampling(std::move(i)); } - auto result = std::move(sampler).samples(); + auto result = sampler.samples(); EXPECT_EQ(5, result.size()); for (auto i : result) { EXPECT_LE(0, i); @@ -27,16 +27,18 @@ TEST(ReservoirSamplingTest, Sample) { } { ReservoirSampling sampler(5); - std::vector sampleSpace = {0, 1, 2}; - for (auto i : sampleSpace) { - sampler.sampling(std::move(i)); - } + for (size_t count = 0; count < 10; count++) { + std::vector sampleSpace = {0, 1, 2}; + for (auto i : sampleSpace) { + sampler.sampling(std::move(i)); + } - auto result = std::move(sampler).samples(); - EXPECT_EQ(3, result.size()); - EXPECT_EQ(0, result[0]); - EXPECT_EQ(1, result[1]); - EXPECT_EQ(2, result[2]); + auto result = sampler.samples(); + EXPECT_EQ(3, result.size()); + EXPECT_EQ(0, result[0]); + EXPECT_EQ(1, result[1]); + EXPECT_EQ(2, result[2]); + } } } } // namespace algorithm diff --git a/src/common/base/test/LoggingBenchmark.cpp b/src/common/base/test/LoggingBenchmark.cpp index de88a432b68..736ba030f1a 100644 --- a/src/common/base/test/LoggingBenchmark.cpp +++ b/src/common/base/test/LoggingBenchmark.cpp @@ -6,6 +6,9 @@ #include #include +#include +#include +#include #include #include @@ -18,6 +21,26 @@ << "123"; \ } +#define XLOG_SOMETHING(iters) \ + for (int64_t i = 0; i < iters; i++) { \ + XLOG(INFO) << "Hello" \ + << " " \ + << "Wolrd" \ + << "123"; \ + } + +class XlogInit { + public: + explicit XlogInit(folly::StringPiece config) { folly::initLogging(config); } +}; + +static void xlogRegistFileHandler() { + folly::LoggerDB::get().registerHandlerFactory(std::make_unique()); + // Since glog outputs the logs to /tmp by default, so we explicitly set a file handler for + // xlog and output logs to /tmp. + folly::initLogging(";default=file:path=/tmp/logging_bm.log"); +} + /*************************** * * native @@ -34,6 +57,9 @@ void loggingUsingGlog(int64_t iters) { LOG_SOMETHING(iters); } #include "common/base/Logging.h" void loggingOptimized(int64_t iters) { LOG_SOMETHING(iters); } +#include +void loggingUsingXlog(int64_t iters) { XLOG_SOMETHING(iters); } + /*************************** * * Run benchmarks @@ -49,6 +75,18 @@ BENCHMARK_RELATIVE(optimized_output_logs, iters) { loggingOptimized(iters); } +BENCHMARK_RELATIVE(xlog_output_logs, iters) { + BENCHMARK_SUSPEND { static XlogInit init(".=INFO:default"); } + loggingUsingXlog(iters); +} + +BENCHMARK_RELATIVE(xlog_output_logs_async, iters) { + BENCHMARK_SUSPEND { + static XlogInit init(".=INFO:default;default:async=true,max_buffer_size=4096"); + } + loggingUsingXlog(iters); +} + BENCHMARK_DRAW_LINE(); BENCHMARK(glog_skip_logs, iters) { @@ -61,6 +99,10 @@ BENCHMARK_RELATIVE(optimized_skip_logs, iters) { loggingOptimized(iters); } +BENCHMARK_RELATIVE(xlog_skip_logs, iters) { + BENCHMARK_SUSPEND { static XlogInit init(".=WARN:default"); } + loggingUsingXlog(iters); +} /*************************** * * main() @@ -68,21 +110,24 @@ BENCHMARK_RELATIVE(optimized_skip_logs, iters) { **************************/ int main(int argc, char** argv) { folly::init(&argc, &argv, true); + xlogRegistFileHandler(); folly::runBenchmarks(); return 0; } /* -Benchmark number is taken from WSL running on i7-8650 - +Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz ============================================================================ -LoggingBenchmark.cpp relative time/iter iters/s +src/common/base/test/LoggingBenchmark.cpp relative time/iter iters/s ============================================================================ -glog_output_logs 3.13us 319.07K -optimized_output_logs 100.10% 3.13us 319.39K +glog_output_logs 1.86us 536.82K +optimized_output_logs 100.26% 1.86us 538.24K +xlog_output_logs 52.73% 3.53us 283.09K +xlog_output_logs_async 53.40% 3.49us 286.68K ---------------------------------------------------------------------------- -glog_skip_logs 1.76us 567.45K -optimized_skip_logs inf% 0.00fs Infinity +glog_skip_logs 1.27us 789.36K +optimized_skip_logs 94753.03% 1.34ns 747.94M +xlog_skip_logs 5215.83% 24.29ns 41.17M ============================================================================ */ diff --git a/src/common/datatypes/DataSet.h b/src/common/datatypes/DataSet.h index b80cec79793..5c42bbc696d 100644 --- a/src/common/datatypes/DataSet.h +++ b/src/common/datatypes/DataSet.h @@ -7,6 +7,8 @@ #ifndef COMMON_DATATYPES_DATASET_H_ #define COMMON_DATATYPES_DATASET_H_ +#include + #include #include #include @@ -153,6 +155,44 @@ struct DataSet { return os.str(); } + // format: + // [ + // { + // "row": [ row-data ], + // "meta": [ metadata ] + // }, + // ] + folly::dynamic toJson() const { + // parse rows to json + auto dataBody = folly::dynamic::array(); + for (auto& row : rows) { + dataBody.push_back(rowToJson(row)); + } + + return dataBody; + } + + // parse Nebula::Row to json + // format: + // { + // "row": [ row-data ], + // "meta": [ metadata ] + // } + folly::dynamic rowToJson(const Row& row) const { + folly::dynamic rowJsonObj = folly::dynamic::object(); + auto rowDataList = folly::dynamic::array(); + auto metaDataList = folly::dynamic::array(); + + for (const auto& ele : row.values) { + rowDataList.push_back(ele.toJson()); + metaDataList.push_back(ele.getMetaData()); + } + + rowJsonObj.insert("row", rowDataList); + rowJsonObj.insert("meta", metaDataList); + return rowJsonObj; + } + bool operator==(const DataSet& rhs) const { return colNames == rhs.colNames && rows == rhs.rows; } }; diff --git a/src/common/datatypes/Date.h b/src/common/datatypes/Date.h index 732fc786eda..7afd65f4549 100644 --- a/src/common/datatypes/Date.h +++ b/src/common/datatypes/Date.h @@ -7,6 +7,8 @@ #ifndef COMMON_DATATYPES_DATE_H_ #define COMMON_DATATYPES_DATE_H_ +#include + #include namespace nebula { @@ -62,6 +64,7 @@ struct Date { Date operator-(int64_t days) const; std::string toString() const; + folly::dynamic toJson() const { return toString(); } // Return the number of days since -32768/1/1 int64_t toInt() const; @@ -113,6 +116,8 @@ struct Time { } std::string toString() const; + // 'Z' representing UTC timezone + folly::dynamic toJson() const { return toString() + "Z"; } }; inline std::ostream& operator<<(std::ostream& os, const Time& d) { @@ -203,6 +208,8 @@ struct DateTime { } std::string toString() const; + // 'Z' representing UTC timezone + folly::dynamic toJson() const { return toString() + "Z"; } }; inline std::ostream& operator<<(std::ostream& os, const DateTime& d) { diff --git a/src/common/datatypes/Edge.cpp b/src/common/datatypes/Edge.cpp index 95d04aa41a6..1893fb79fca 100644 --- a/src/common/datatypes/Edge.cpp +++ b/src/common/datatypes/Edge.cpp @@ -32,6 +32,45 @@ std::string Edge::toString() const { return os.str(); } +// format: +// { +// "prop1": val1, +// "prop2": val2, +// } +folly::dynamic Edge::toJson() const { + folly::dynamic propObj = folly::dynamic::object(); + + for (const auto& iter : props) { + propObj.insert(iter.first, iter.second.toJson()); + } + + return propObj; +} + +// Used in Json form query result +// format: +// { +// "id": { +// "name": _name, +// "src": srcVID, +// "dst": dstVID, +// "type": _type, +// "ranking": _rankding +// } +// "type": "edge" +// } +folly::dynamic Edge::getMetaData() const { + folly::dynamic edgeMetadataObj = folly::dynamic::object(); + + folly::dynamic edgeIdObj = folly::dynamic::object("name", name)("src", src.toJson())( + "dst", dst.toJson())("type", type)("ranking", ranking); + + edgeMetadataObj.insert("id", edgeIdObj); + edgeMetadataObj.insert("type", "edge"); + + return edgeMetadataObj; +} + bool Edge::contains(const Value& key) const { if (!key.isStr()) { return false; diff --git a/src/common/datatypes/Edge.h b/src/common/datatypes/Edge.h index 32440217eda..b1c3247af84 100644 --- a/src/common/datatypes/Edge.h +++ b/src/common/datatypes/Edge.h @@ -50,6 +50,9 @@ struct Edge { void __clear() { clear(); } std::string toString() const; + folly::dynamic toJson() const; + // Used in Json form query result + folly::dynamic getMetaData() const; bool operator==(const Edge& rhs) const; diff --git a/src/common/datatypes/List.cpp b/src/common/datatypes/List.cpp index ca747dee6b9..ac2af1679ff 100644 --- a/src/common/datatypes/List.cpp +++ b/src/common/datatypes/List.cpp @@ -22,4 +22,24 @@ std::string List::toString() const { return os.str(); } +folly::dynamic List::toJson() const { + auto listJsonObj = folly::dynamic::array(); + + for (const auto& val : values) { + listJsonObj.push_back(val.toJson()); + } + + return listJsonObj; +} + +folly::dynamic List::getMetaData() const { + auto listMetadataObj = folly::dynamic::array(); + + for (const auto& val : values) { + listMetadataObj.push_back(val.getMetaData()); + } + + return listMetadataObj; +} + } // namespace nebula diff --git a/src/common/datatypes/List.h b/src/common/datatypes/List.h index c5622820905..be768a0de08 100644 --- a/src/common/datatypes/List.h +++ b/src/common/datatypes/List.h @@ -65,6 +65,9 @@ struct List { size_t size() const { return values.size(); } std::string toString() const; + folly::dynamic toJson() const; + // Extract the metadata of each element + folly::dynamic getMetaData() const; }; inline std::ostream& operator<<(std::ostream& os, const List& l) { return os << l.toString(); } diff --git a/src/common/datatypes/Map.cpp b/src/common/datatypes/Map.cpp index b9839c120ae..7ab73d5c9b7 100644 --- a/src/common/datatypes/Map.cpp +++ b/src/common/datatypes/Map.cpp @@ -14,7 +14,7 @@ namespace nebula { std::string Map::toString() const { std::vector value(kvs.size()); - std::transform(kvs.begin(), kvs.end(), value.begin(), [](const auto &iter) -> std::string { + std::transform(kvs.begin(), kvs.end(), value.begin(), [](const auto& iter) -> std::string { std::stringstream out; out << iter.first << ":" << iter.second; return out.str(); @@ -25,4 +25,24 @@ std::string Map::toString() const { return os.str(); } +folly::dynamic Map::toJson() const { + folly::dynamic mapJsonObj = folly::dynamic::object(); + + for (const auto& iter : kvs) { + mapJsonObj.insert(iter.first, iter.second.toJson()); + } + + return mapJsonObj; +} + +folly::dynamic Map::getMetaData() const { + auto mapMetadataObj = folly::dynamic::array(); + + for (const auto& kv : kvs) { + mapMetadataObj.push_back(kv.second.getMetaData()); + } + + return mapMetadataObj; +} + } // namespace nebula diff --git a/src/common/datatypes/Map.h b/src/common/datatypes/Map.h index 7f7ec9b6a81..333e5d0cb0d 100644 --- a/src/common/datatypes/Map.h +++ b/src/common/datatypes/Map.h @@ -43,6 +43,9 @@ struct Map { // the configs of rocksdb will use the interface, so the value need modify to // string std::string toString() const; + folly::dynamic toJson() const; + // Extract the metadata of the value of each kv pair + folly::dynamic getMetaData() const; bool operator==(const Map& rhs) const { return kvs == rhs.kvs; } diff --git a/src/common/datatypes/Path.h b/src/common/datatypes/Path.h index 1d4e2184ded..17eb77a3cee 100644 --- a/src/common/datatypes/Path.h +++ b/src/common/datatypes/Path.h @@ -130,6 +130,55 @@ struct Path { return os.str(); } + folly::dynamic toJson() const { + folly::dynamic pathJsonObj = folly::dynamic::array(); + auto srcVertex = src; + pathJsonObj.push_back(srcVertex.toJson()); + + for (const auto& s : steps) { + folly::dynamic edgeJsonObj = folly::dynamic::object(); + // parse edge props map as json + for (const auto& iter : s.props) { + edgeJsonObj.insert(iter.first, iter.second.toJson()); + } + // add edge json obj to path + pathJsonObj.push_back(edgeJsonObj); + + // reset src vertex and add vertex json obj to path + srcVertex = s.dst; + pathJsonObj.push_back(srcVertex.toJson()); + } + + return pathJsonObj; + } + + // Used in Json form query result + // format: + // [vertex1_metadata, edge1_metadata, vertex2_metadata, edge2_metadata,....] + folly::dynamic getMetaData() const { + auto dynamicObj = folly::dynamic::array(); + auto srcVertex = src; + dynamicObj.push_back(srcVertex.getMetaData()); + + // Construct edge metadata + for (const auto& s : steps) { + folly::dynamic edgeIdObj = folly::dynamic::object(); + edgeIdObj.insert("src", srcVertex.vid.toJson()); + edgeIdObj.insert("dst", s.dst.vid.toJson()); + edgeIdObj.insert("type", s.type); + edgeIdObj.insert("name", s.name); + edgeIdObj.insert("ranking", s.ranking); + + folly::dynamic edgeMetadataObj = folly::dynamic::object("id", edgeIdObj)("type", "edge"); + dynamicObj.push_back(edgeMetadataObj); + dynamicObj.push_back(s.dst.getMetaData()); + // reset src vertex + srcVertex = s.dst; + } + + return dynamicObj; + } + Path& operator=(Path&& rhs) noexcept { if (&rhs != this) { src = std::move(rhs.src); diff --git a/src/common/datatypes/Set.cpp b/src/common/datatypes/Set.cpp index c31ffc1de21..2e0e90a2f5a 100644 --- a/src/common/datatypes/Set.cpp +++ b/src/common/datatypes/Set.cpp @@ -22,4 +22,24 @@ std::string Set::toString() const { return os.str(); } +folly::dynamic Set::toJson() const { + auto setJsonObj = folly::dynamic::array(); + + for (const auto& val : values) { + setJsonObj.push_back(val.toJson()); + } + + return setJsonObj; +} + +folly::dynamic Set::getMetaData() const { + auto setMetadataObj = folly::dynamic::array(); + + for (const auto& val : values) { + setMetadataObj.push_back(val.getMetaData()); + } + + return setMetadataObj; +} + } // namespace nebula diff --git a/src/common/datatypes/Set.h b/src/common/datatypes/Set.h index e844aa879eb..dabc33e0d7e 100644 --- a/src/common/datatypes/Set.h +++ b/src/common/datatypes/Set.h @@ -26,6 +26,9 @@ struct Set { void __clear() { clear(); } std::string toString() const; + folly::dynamic toJson() const; + // Extract the metadata of each element + folly::dynamic getMetaData() const; Set& operator=(const Set& rhs) { if (this == &rhs) { diff --git a/src/common/datatypes/Value.cpp b/src/common/datatypes/Value.cpp index ad6e82e4043..19545de8135 100644 --- a/src/common/datatypes/Value.cpp +++ b/src/common/datatypes/Value.cpp @@ -1344,6 +1344,116 @@ void Value::setG(DataSet&& v) { new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(std::move(v))); } +// Convert Nebula::Value to a value compatible with Json standard +// DATE, TIME, DATETIME will be converted to strings in UTC +// VERTEX, EDGES, PATH will be converted to objects +folly::dynamic Value::toJson() const { + switch (type_) { + case Value::Type::__EMPTY__: { + return "__EMPTY__"; + } + // Json null + case Value::Type::NULLVALUE: { + return folly::dynamic(nullptr); + } + // Json bool + case Value::Type::BOOL: { + return folly::dynamic(getBool()); + } + // Json int + case Value::Type::INT: { + return folly::dynamic(getInt()); + } + // json double + case Value::Type::FLOAT: { + return folly::dynamic(getFloat()); + } + // json string + case Value::Type::STRING: { + return folly::dynamic(getStr()); + } + // Json array + case Value::Type::LIST: { + return getList().toJson(); + } + case Value::Type::SET: { + return getSet().toJson(); + } + // Json object + case Value::Type::MAP: { + return getMap().toJson(); + } + case Value::Type::DATE: { + return getDate().toJson(); + } + case Value::Type::TIME: { + return getTime().toJson(); + } + case Value::Type::DATETIME: { + return getDateTime().toJson(); + } + case Value::Type::EDGE: { + return getEdge().toJson(); + } + case Value::Type::VERTEX: { + return getVertex().toJson(); + } + case Value::Type::PATH: { + return getPath().toJson(); + } + case Value::Type::DATASET: { + return getDataSet().toJson(); + } + // no default so the compiler will warning when lack + } + + LOG(FATAL) << "Unknown value type " << static_cast(type_); +} + +folly::dynamic Value::getMetaData() const { + auto dynamicObj = folly::dynamic(); + switch (type_) { + // Privative datatypes has no meta data + case Value::Type::__EMPTY__: + case Value::Type::BOOL: + case Value::Type::INT: + case Value::Type::FLOAT: + case Value::Type::STRING: + case Value::Type::DATASET: + case Value::Type::NULLVALUE: { + return folly::dynamic(nullptr); + } + // Extract the meta info of each element as the metadata of the container + case Value::Type::LIST: { + return getList().getMetaData(); + } + case Value::Type::SET: { + return getSet().getMetaData(); + } + case Value::Type::MAP: { + return getMap().getMetaData(); + } + case Value::Type::DATE: + case Value::Type::TIME: + case Value::Type::DATETIME: { + return folly::dynamic::object("type", typeName()); + } + case Value::Type::VERTEX: { + return getVertex().getMetaData(); + } + case Value::Type::EDGE: { + return getEdge().getMetaData(); + } + case Value::Type::PATH: { + return getPath().getMetaData(); + } + default: + break; + } + + LOG(FATAL) << "Unknown value type " << static_cast(type_); +} + std::string Value::toString() const { switch (type_) { case Value::Type::__EMPTY__: { diff --git a/src/common/datatypes/Value.h b/src/common/datatypes/Value.h index 00c413cb4a9..57dc60df4e4 100644 --- a/src/common/datatypes/Value.h +++ b/src/common/datatypes/Value.h @@ -7,6 +7,8 @@ #ifndef COMMON_DATATYPES_VALUE_H_ #define COMMON_DATATYPES_VALUE_H_ +#include + #include #include "common/datatypes/Date.h" @@ -268,6 +270,9 @@ struct Value { static const Value& null() noexcept { return kNullValue; } std::string toString() const; + folly::dynamic toJson() const; + // Used in Json form query result + folly::dynamic getMetaData() const; Value toBool() const; Value toFloat() const; diff --git a/src/common/datatypes/Vertex.cpp b/src/common/datatypes/Vertex.cpp index 1d0f227025e..9f839a26831 100644 --- a/src/common/datatypes/Vertex.cpp +++ b/src/common/datatypes/Vertex.cpp @@ -6,6 +6,7 @@ #include "common/datatypes/Vertex.h" +#include #include #include @@ -24,6 +25,20 @@ std::string Tag::toString() const { return os.str(); } +// { +// "player.name" : "Tim Duncan", +// "player.age" : 42, +// } +folly::dynamic Tag::toJson() const { + folly::dynamic tagJsonObj = folly::dynamic::object(); + + for (const auto& iter : props) { + tagJsonObj.insert(name + "." + iter.first, iter.second.toJson()); + } + + return tagJsonObj; +} + bool Vertex::contains(const Value& key) const { if (!key.isStr()) { return false; @@ -59,6 +74,31 @@ std::string Vertex::toString() const { return os.str(); } +// { +// "player.name" : "Tim Duncan", +// "player.age" : 42, +// "bachelor.name" : "Tim Duncan", +// "bachelor.speciality" : "psychology" +// } +folly::dynamic Vertex::toJson() const { + folly::dynamic propJsonObj = folly::dynamic::object(); + + for (const auto& tag : tags) { + propJsonObj.update(tag.toJson()); + } + return propJsonObj; +} + +// format: +// { +// "id": _vid +// "type": "vertex" +// } +folly::dynamic Vertex::getMetaData() const { + folly::dynamic vertexMetadataObj = folly::dynamic::object("id", vid.toJson())("type", "vertex"); + return vertexMetadataObj; +} + Vertex& Vertex::operator=(Vertex&& rhs) noexcept { if (&rhs != this) { vid = std::move(rhs.vid); diff --git a/src/common/datatypes/Vertex.h b/src/common/datatypes/Vertex.h index a0c60318692..08d50b40f0e 100644 --- a/src/common/datatypes/Vertex.h +++ b/src/common/datatypes/Vertex.h @@ -34,6 +34,7 @@ struct Tag { void __clear() { clear(); } std::string toString() const; + folly::dynamic toJson() const; Tag& operator=(Tag&& rhs) noexcept { if (&rhs != this) { @@ -71,6 +72,9 @@ struct Vertex { void __clear() { clear(); } std::string toString() const; + folly::dynamic toJson() const; + // Used in Json form query result + folly::dynamic getMetaData() const; Vertex& operator=(Vertex&& rhs) noexcept; diff --git a/src/common/datatypes/test/CMakeLists.txt b/src/common/datatypes/test/CMakeLists.txt index a45c5d2ffd6..54517fda196 100644 --- a/src/common/datatypes/test/CMakeLists.txt +++ b/src/common/datatypes/test/CMakeLists.txt @@ -79,6 +79,25 @@ nebula_add_test( gtest ) +nebula_add_test( + NAME + value_to_json_test + SOURCES + ValueToJsonTest.cpp + OBJECTS + $ + $ + $ + $ + $ + $ + $ + $ + LIBRARIES + gtest + ${THRIFT_LIBRARIES} +) + nebula_add_executable( NAME edge_bm diff --git a/src/common/datatypes/test/ValueToJsonTest.cpp b/src/common/datatypes/test/ValueToJsonTest.cpp new file mode 100644 index 00000000000..3f729d37a6b --- /dev/null +++ b/src/common/datatypes/test/ValueToJsonTest.cpp @@ -0,0 +1,314 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ +#include +#include + +#include "common/base/Base.h" +#include "common/datatypes/CommonCpp2Ops.h" +#include "common/datatypes/DataSet.h" +#include "common/datatypes/Date.h" +#include "common/datatypes/Edge.h" +#include "common/datatypes/List.h" +#include "common/datatypes/Map.h" +#include "common/datatypes/Path.h" +#include "common/datatypes/Set.h" +#include "common/datatypes/Value.h" +#include "common/datatypes/Vertex.h" + +namespace nebula { + +using folly::dynamic; +using serializer = apache::thrift::CompactSerializer; + +TEST(ValueToJson, vertex) { + // Test tag to json + auto tag1 = Tag("tagName", {{"prop", Value(2)}}); + auto tag2 = + Tag("tagName1", + {{"prop1", Value(2)}, {"prop2", Value(NullType::__NULL__)}, {"prop3", Value("123")}}); + { + dynamic expectedTagJson = dynamic::object("tagName.prop", 2); + ASSERT_EQ(expectedTagJson, tag1.toJson()); + } + { + dynamic expectedTagJson = + dynamic::object("tagName1.prop1", 2)("tagName1.prop2", nullptr)("tagName1.prop3", "123"); + ASSERT_EQ(expectedTagJson, tag2.toJson()); + } + + // vertex wtih string vid + auto vertexStrVid = Value(Vertex({"Vid", + { + tag1, + tag2, + }})); + + // integerID vertex + auto vertexIntVid = Value(Vertex({001, + { + tag1, + tag2, + }})); + { + dynamic expectedVeretxJson = dynamic::object("tagName.prop", 2)("tagName1.prop1", 2)( + "tagName1.prop2", nullptr)("tagName1.prop3", "123"); + ASSERT_EQ(expectedVeretxJson, vertexStrVid.toJson()); + + dynamic expectedVeretxMetaJson = dynamic::object("id", "Vid")("type", "vertex"); + ASSERT_EQ(expectedVeretxMetaJson, vertexStrVid.getMetaData()); + } + { + dynamic expectedVeretxJson = dynamic::object("tagName.prop", 2)("tagName1.prop1", 2)( + "tagName1.prop2", nullptr)("tagName1.prop3", "123"); + ASSERT_EQ(expectedVeretxJson, vertexIntVid.toJson()); + + dynamic expectedVeretxMetaJson = dynamic::object("id", 001)("type", "vertex"); + ASSERT_EQ(expectedVeretxMetaJson, vertexIntVid.getMetaData()); + } +} + +TEST(ValueToJson, edge) { + // edge + auto edge1 = + Value(Edge("Src", "Dst", 1, "Edge", 233, {{"prop1", Value(233)}, {"prop2", Value(2.3)}})); + // integerID edge + auto edge2 = + Value(Edge(101, 102, 1, "Edge", 233, {{"prop1", Value(233)}, {"prop2", Value(2.3)}})); + { + dynamic expectedEdgeJson = dynamic::object("prop1", 233)("prop2", 2.3); + ASSERT_EQ(expectedEdgeJson, edge1.toJson()); + + dynamic expectedEdgeMetaJson = + dynamic::object("id", + dynamic::object("name", "Edge")("src", "Src")("dst", "Dst")("type", 1)( + "name", "Edge")("ranking", 233))("type", "edge"); + ASSERT_EQ(expectedEdgeMetaJson, edge1.getMetaData()); + } + + { + dynamic expectedEdgeJson = dynamic::object("prop1", 233)("prop2", 2.3); + ASSERT_EQ(expectedEdgeJson, edge2.toJson()); + + dynamic expectedEdgeMetaJson = + dynamic::object("id", + dynamic::object("name", "Edge")("src", 101)("dst", 102)("type", 1)( + "name", "Edge")("ranking", 233))("type", "edge"); + ASSERT_EQ(expectedEdgeMetaJson, edge2.getMetaData()); + } +} + +TEST(ValueToJson, path) { + auto path = Value(Path(Vertex({"v1", {Tag("tagName", {{"prop1", Value(1)}})}}), + {Step(Vertex({"v2", + {Tag("tagName2", + {{"prop1", Value(2)}, + {"prop2", Value(NullType::__NULL__)}, + {"prop3", Value("123")}})}}), + 1, + "edgeName", + 100, + {{"edgeProp", "edgePropVal"}})})); + auto emptyPath = Value(Path()); + + dynamic expectedPathJsonObj = dynamic::array( + dynamic::object("tagName.prop1", 1), + dynamic::object("edgeProp", "edgePropVal"), + dynamic::object("tagName2.prop1", 2)("tagName2.prop2", nullptr)("tagName2.prop3", "123")); + ASSERT_EQ(expectedPathJsonObj, path.toJson()); + + dynamic expectedPathMetaJson = dynamic::array( + dynamic::object("id", "v1")("type", "vertex"), + dynamic::object("id", + dynamic::object("name", "Edge")("src", "v1")("dst", "v2")("type", 1)( + "name", "edgeName")("ranking", 100))("type", "edge"), + dynamic::object("id", "v2")("type", "vertex")); + ASSERT_EQ(expectedPathMetaJson, path.getMetaData()); +} + +TEST(ValueToJson, list) { + auto list1 = Value(List({Value(2), // int + Value(2.33), // float + Value(true), // bool + Value("str"), // string + Date(2021, 12, 21), // date + Time(13, 30, 15, 0), // time + DateTime(2021, 12, 21, 13, 30, 15, 0)})); // datetime + dynamic expectedListJsonObj = dynamic::array( + 2, 2.33, true, "str", "2021-12-21", "13:30:15.000000Z", "2021-12-21T13:30:15.0Z"); + ASSERT_EQ(expectedListJsonObj, list1.toJson()); + + dynamic expectedListMetaObj = dynamic::array(nullptr, + nullptr, + nullptr, + nullptr, + dynamic::object("type", "date"), + dynamic::object("type", "time"), + dynamic::object("type", "datetime")); + ASSERT_EQ(expectedListMetaObj, list1.getMetaData()); +} + +TEST(ValueToJson, Set) { + auto set = Value(Set({Value(2), // int + Value(2.33), // float + Value(true), // bool + Value("str"), // string + Date(2021, 12, 21), // date + Time(13, 30, 15, 0), // time + DateTime(2021, 12, 21, 13, 30, 15, 0)})); // datetime + dynamic expectedSetJsonObj = dynamic::array( + 2, 2.33, true, "str", "2021-12-21", "13:30:15.000000Z", "2021-12-21T13:30:15.0Z"); + // The underlying data strcuture is unordered_set, so sort before the comparison + auto actualJson = set.toJson(); + std::sort(actualJson.begin(), actualJson.end()); + std::sort(expectedSetJsonObj.begin(), expectedSetJsonObj.end()); + ASSERT_EQ(expectedSetJsonObj, actualJson); + + // Skip meta json comparison since nested dynamic objects cannot be sorted. i.g. dynamic::object + // inside dynamic::array +} + +TEST(ValueToJson, map) { + auto map = Value(Map({{"key1", Value(2)}, // int + {"key2", Value(2.33)}, // float + {"key3", Value(true)}, // bool + {"key4", Value("str")}, // string + {"key5", Date(2021, 12, 21)}, // date + {"key6", Time(13, 30, 15, 0)}, // time + {"key7", DateTime(2021, 12, 21, 13, 30, 15, 0)}})); // datetime + dynamic expectedMapJsonObj = + dynamic::object("key1", 2)("key2", 2.33)("key3", true)("key4", "str")("key5", "2021-12-21")( + "key6", "13:30:15.000000Z")("key7", "2021-12-21T13:30:15.0Z"); + ASSERT_EQ(expectedMapJsonObj, map.toJson()); + // Skip meta json comparison since nested dynamic objects cannot be sorted. i.g. dynamic::object + // inside dynamic::array +} + +TEST(ValueToJson, dataset) { + DataSet dataset = DataSet({"col1", "col2", "col3", "col4", "col5", "col6", "col7"}); + dataset.emplace_back(List({Value(2), // int + Value(2.33), // float + Value(true), // bool + Value("str"), // string + Date(2021, 12, 21), // date + Time(13, 30, 15, 0), // time + DateTime(2021, 12, 21, 13, 30, 15, 0)})); + dynamic expectedDatasetJsonObj = dynamic::array(dynamic::object( + "row", + dynamic::array( + 2, 2.33, true, "str", "2021-12-21", "13:30:15.000000Z", "2021-12-21T13:30:15.0Z"))( + "meta", + dynamic::array(nullptr, + nullptr, + nullptr, + nullptr, + dynamic::object("type", "date"), + dynamic::object("type", "time"), + dynamic::object("type", "datetime")))); + ASSERT_EQ(expectedDatasetJsonObj, dataset.toJson()); +} + +TEST(ValueToJson, DecodeEncode) { + std::vector values{ + // empty + Value(), + + // null + Value(NullType::__NULL__), + Value(NullType::DIV_BY_ZERO), + Value(NullType::BAD_DATA), + Value(NullType::ERR_OVERFLOW), + Value(NullType::OUT_OF_RANGE), + Value(NullType::UNKNOWN_PROP), + + // int + Value(0), + Value(1), + Value(2), + + // float + Value(3.14), + Value(2.67), + + // string + Value("Hello "), + Value("World"), + + // bool + Value(false), + Value(true), + + // date + Value(Date(2020, 1, 1)), + Value(Date(2019, 12, 1)), + + // time + Value(Time{1, 2, 3, 4}), + + // datatime + Value(DateTime{1, 2, 3, 4, 5, 6, 7}), + + // vertex + Value( + Vertex({"Vid", + { + Tag("tagName", {{"prop", Value(2)}}), + Tag("tagName1", {{"prop1", Value(2)}, {"prop2", Value(NullType::__NULL__)}}), + }})), + + // integerID vertex + Value( + Vertex({001, + { + Tag("tagName", {{"prop", Value(2)}}), + Tag("tagName1", {{"prop1", Value(2)}, {"prop2", Value(NullType::__NULL__)}}), + }})), + + // edge + Value(Edge("Src", "Dst", 1, "Edge", 233, {{"prop1", Value(233)}, {"prop2", Value(2.3)}})), + + // integerID edge + Value(Edge(001, 002, 1, "Edge", 233, {{"prop1", Value(233)}, {"prop2", Value(2.3)}})), + + // Path + Value(Path( + Vertex({"1", {Tag("tagName", {{"prop", Value(2)}})}}), + {Step(Vertex({"1", {Tag("tagName", {{"prop", Value(2)}})}}), 1, "1", 1, {{"1", 1}})})), + Value(Path()), + + // List + Value(List({Value(2), Value(true), Value(2.33)})), + + // Set + Value(Set({Value(2), Value(true), Value(2.33)})), + + // Map + Value(Map({{"Key1", Value(2)}, {"Key2", Value(true)}, {"Key3", Value(2.33)}})), + + // DataSet + Value(DataSet({"col1", "col2"})), + }; + for (const auto& val : values) { + std::string buf; + buf.reserve(128); + folly::dynamic jsonObj = val.toJson(); + auto jsonString = folly::toJson(jsonObj); + serializer::serialize(jsonString, &buf); + std::string valCopy; + std::size_t s = serializer::deserialize(buf, valCopy); + ASSERT_EQ(s, buf.size()); + EXPECT_EQ(jsonString, valCopy); + } +} + +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} diff --git a/src/common/graph/Response.cpp b/src/common/graph/Response.cpp index a03885bd1fa..82f3def31c7 100644 --- a/src/common/graph/Response.cpp +++ b/src/common/graph/Response.cpp @@ -31,7 +31,7 @@ bool PlanNodeDescription::operator==(const PlanNodeDescription& rhs) const { case ErrorCode::EnumName: \ return #EnumName; -const char* errorCode(ErrorCode code) { +const char* getErrorCode(ErrorCode code) { switch (code) { ErrorCodeEnums } return "Unknown error"; } diff --git a/src/common/graph/Response.h b/src/common/graph/Response.h index 0adfd2b52cf..3d7ee8b88c1 100644 --- a/src/common/graph/Response.h +++ b/src/common/graph/Response.h @@ -7,6 +7,9 @@ #ifndef COMMON_GRAPH_RESPONSE_H #define COMMON_GRAPH_RESPONSE_H +#include +#include + #include #include #include @@ -183,10 +186,10 @@ enum class ErrorCode { ErrorCodeEnums }; #undef X -const char *errorCode(ErrorCode code); +const char *getErrorCode(ErrorCode code); static inline std::ostream &operator<<(std::ostream &os, ErrorCode code) { - os << errorCode(code); + os << getErrorCode(code); return os; } @@ -266,6 +269,16 @@ struct ProfilingStats { int64_t totalDurationInUs{0}; // Other profiling stats data map std::unique_ptr> otherStats; + + folly::dynamic toJson() const { + folly::dynamic ProfilingStatsObj = folly::dynamic::object(); + ProfilingStatsObj.insert("rows", rows); + ProfilingStatsObj.insert("execDurationInUs", execDurationInUs); + ProfilingStatsObj.insert("totalDurationInUs", totalDurationInUs); + ProfilingStatsObj.insert("otherStats", folly::toDynamic(*otherStats)); + + return ProfilingStatsObj; + } }; // The info used for select/loop. @@ -285,6 +298,14 @@ struct PlanNodeBranchInfo { bool isDoBranch{0}; // select/loop node id int64_t conditionNodeId{-1}; + + folly::dynamic toJson() const { + folly::dynamic PlanNodeBranchInfoObj = folly::dynamic::object(); + PlanNodeBranchInfoObj.insert("isDoBranch", isDoBranch); + PlanNodeBranchInfoObj.insert("conditionNodeId", conditionNodeId); + + return PlanNodeBranchInfoObj; + } }; struct Pair { @@ -299,6 +320,11 @@ struct Pair { std::string key; std::string value; + + folly::dynamic toJson() const { + folly::dynamic pairObj = folly::dynamic::object(key, value); + return pairObj; + } }; struct PlanNodeDescription { @@ -326,6 +352,32 @@ struct PlanNodeDescription { std::unique_ptr> profiles{nullptr}; std::unique_ptr branchInfo{nullptr}; std::unique_ptr> dependencies{nullptr}; + + folly::dynamic toJson() const { + folly::dynamic planNodeDescObj = folly::dynamic::object(); + planNodeDescObj.insert("name", name); + planNodeDescObj.insert("id", id); + planNodeDescObj.insert("outputVar", outputVar); + + auto descriptionObj = folly::dynamic::array(); + descriptionObj.resize(description->size()); + std::transform( + description->begin(), description->end(), descriptionObj.begin(), [](const auto &ele) { + return ele.toJson(); + }); + planNodeDescObj.insert("description", descriptionObj); + + auto profilesObj = folly::dynamic::array(); + profilesObj.resize(profiles->size()); + std::transform(profiles->begin(), profiles->end(), profilesObj.begin(), [](const auto &ele) { + return ele.toJson(); + }); + planNodeDescObj.insert("profiles", profilesObj); + planNodeDescObj.insert("branchInfo", branchInfo->toJson()); + planNodeDescObj.insert("dependencies", folly::toDynamic(*dependencies)); + + return planNodeDescObj; + } }; struct PlanDescription { @@ -350,6 +402,29 @@ struct PlanDescription { std::string format; // the optimization spent time int32_t optimize_time_in_us{0}; + + folly::dynamic toJson() const { + folly::dynamic PlanDescObj = folly::dynamic::object(); + + auto planNodeDescsObj = folly::dynamic::array(); + planNodeDescsObj.resize(planNodeDescs.size()); + std::transform(planNodeDescs.begin(), + planNodeDescs.end(), + planNodeDescsObj.begin(), + [](const PlanNodeDescription &ele) { return ele.toJson(); }); + PlanDescObj.insert("planNodeDescs", planNodeDescsObj); + // nodeIndexMap uses int as the key of the map, but strict json format only accepts string as + // the key, so convert the int to string here. + folly::dynamic nodeIndexMapObj = folly::dynamic::object(); + for (const auto &kv : nodeIndexMap) { + nodeIndexMapObj.insert(folly::to(kv.first), kv.second); + } + PlanDescObj.insert("nodeIndexMap", nodeIndexMapObj); + PlanDescObj.insert("format", format); + PlanDescObj.insert("optimize_time_in_us", optimize_time_in_us); + + return PlanDescObj; + } }; struct ExecutionResponse { @@ -397,6 +472,83 @@ struct ExecutionResponse { std::unique_ptr errorMsg{nullptr}; std::unique_ptr planDesc{nullptr}; std::unique_ptr comment{nullptr}; + + // Return the response as a json string + // format + // "results": [ + // { + // "columns": [], + // "data": [ + // { + // "row": [ row-data ], + // "meta": [ metadata ] + // }, + // ], + // "latencyInUs" : 0, + // "spaceName": "", + // "planDesc ": { + // "planNodeDescs": [ { + // "name" : "", + // "id" : 0, + // "outputVar" : "", + // "description" : {"key" : ""}, + // "profiles" : [{ + // "rows" : 1, + // "execDurationInUs" : 0, + // "totalDurationInUs" : 0, + // "otherStats" : {}, // map + // }], + // "branchInfo" : { + // "isDoBranch" : false, + // "conditionNodeId" : -1, + // }, + // "dependencies" : [] // vector of ints + // } + // ], + // "nodeIndexMap" : {}, + // "format" : "", + // "optimize_time_in_us" : 0, + // }, + // "comment ": "", + // "errors" : "" // errorMsg + // } + // ] + // } + folly::dynamic toJson() const { + folly::dynamic respJsonObj = folly::dynamic::object(); + folly::dynamic resultBody = folly::dynamic::object(); + + // required fields + folly::dynamic errorsBody = folly::dynamic::object(); + errorsBody.insert("errorCode", getErrorCode(errorCode)); + resultBody.insert("latencyInUs", latencyInUs); + + // optional fields + if (errorMsg) { + errorsBody.insert("errorMsg", *errorMsg); + } + resultBody.insert("errors", errorsBody); + + if (data) { + resultBody.insert("columns", folly::toDynamic(data->keys())); + resultBody.insert("data", data->toJson()); + } + if (spaceName) { + resultBody.insert("spaceName", *spaceName); + } + if (planDesc) { + resultBody.insert("planDesc", planDesc->toJson()); + } + if (comment) { + resultBody.insert("comment", *comment); + } + + auto resultArray = folly::dynamic::array(); + resultArray.push_back(resultBody); + respJsonObj.insert("results", resultArray); + + return respJsonObj; + } }; } // namespace nebula diff --git a/src/common/graph/tests/ResponseEncodeDecodeTest.cpp b/src/common/graph/tests/ResponseEncodeDecodeTest.cpp index 378525e0617..ec0a024df8c 100644 --- a/src/common/graph/tests/ResponseEncodeDecodeTest.cpp +++ b/src/common/graph/tests/ResponseEncodeDecodeTest.cpp @@ -89,4 +89,57 @@ TEST(ResponseEncodDecodeTest, Basic) { } } +TEST(ResponseEncodDecodeTest, ToJson) { + // plan description + { + std::vector pds; + pds.emplace_back(PlanDescription{}); + pds.emplace_back(PlanDescription{std::vector{}, + std::unordered_map{{1, 2}, {4, 7}}, + "format"}); + for (const auto &pd : pds) { + std::string buf; + buf.reserve(128); + folly::dynamic jsonObj = pd.toJson(); + auto jsonString = folly::toJson(jsonObj); + serializer::serialize(jsonString, &buf); + std::string copy; + std::size_t s = serializer::deserialize(buf, copy); + ASSERT_EQ(s, buf.size()); + EXPECT_EQ(jsonString, copy); + } + } + // response + { + std::vector resps; + resps.emplace_back(ExecutionResponse{}); + resps.emplace_back(ExecutionResponse{ErrorCode::SUCCEEDED, 233}); + resps.emplace_back(ExecutionResponse{ErrorCode::SUCCEEDED, + 233, + std::make_unique(), + std::make_unique("test_space")}); + resps.emplace_back(ExecutionResponse{ErrorCode::SUCCEEDED, + 233, + nullptr, + std::make_unique("test_space"), + nullptr, + std::make_unique()}); + resps.emplace_back(ExecutionResponse{ErrorCode::E_SYNTAX_ERROR, + 233, + nullptr, + std::make_unique("test_space"), + std::make_unique("Error Msg.")}); + for (const auto &resp : resps) { + std::string buf; + buf.reserve(128); + folly::dynamic jsonObj = resp.toJson(); + auto jsonString = folly::toJson(jsonObj); + serializer::serialize(jsonString, &buf); + std::string copy; + std::size_t s = serializer::deserialize(buf, copy); + ASSERT_EQ(s, buf.size()); + EXPECT_EQ(jsonString, copy); + } + } +} } // namespace nebula diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt index 65439480f29..ce7ec27cb95 100644 --- a/src/daemons/CMakeLists.txt +++ b/src/daemons/CMakeLists.txt @@ -117,7 +117,6 @@ nebula_add_executable( $ $ $ - $ $ $ $ diff --git a/src/graph/executor/admin/SubmitJobExecutor.cpp b/src/graph/executor/admin/SubmitJobExecutor.cpp index 8353783b598..592acdc0f0f 100644 --- a/src/graph/executor/admin/SubmitJobExecutor.cpp +++ b/src/graph/executor/admin/SubmitJobExecutor.cpp @@ -35,87 +35,97 @@ folly::Future SubmitJobExecutor::execute() { LOG(ERROR) << resp.status().toString(); return std::move(resp).status(); } - switch (jobOp) { - case meta::cpp2::AdminJobOp::ADD: { - nebula::DataSet v({"New Job Id"}); - DCHECK(resp.value().job_id_ref().has_value()); - if (!resp.value().job_id_ref().has_value()) { - return Status::Error("Response unexpected."); - } - v.emplace_back(nebula::Row({*resp.value().job_id_ref()})); - return finish(std::move(v)); - } - case meta::cpp2::AdminJobOp::RECOVER: { - nebula::DataSet v({"Recovered job num"}); - DCHECK(resp.value().recovered_job_num_ref().has_value()); - if (!resp.value().recovered_job_num_ref().has_value()) { - return Status::Error("Response unexpected."); - } - v.emplace_back(nebula::Row({*resp.value().recovered_job_num_ref()})); - return finish(std::move(v)); - } - case meta::cpp2::AdminJobOp::SHOW: { - nebula::DataSet v( - {"Job Id(TaskId)", "Command(Dest)", "Status", "Start Time", "Stop Time"}); - DCHECK(resp.value().job_desc_ref().has_value()); - if (!resp.value().job_desc_ref().has_value()) { - return Status::Error("Response unexpected."); - } - DCHECK(resp.value().task_desc_ref().has_value()); - if (!resp.value().task_desc_ref().has_value()) { - return Status::Error("Response unexpected"); - } - auto &jobDesc = *resp.value().job_desc_ref(); - // job desc - v.emplace_back(nebula::Row({ - jobDesc.front().get_id(), - apache::thrift::util::enumNameSafe(jobDesc.front().get_cmd()), - apache::thrift::util::enumNameSafe(jobDesc.front().get_status()), - time::TimeConversion::unixSecondsToDateTime(jobDesc.front().get_start_time()), - time::TimeConversion::unixSecondsToDateTime(jobDesc.front().get_stop_time()), - })); - // tasks desc - auto &tasksDesc = *resp.value().get_task_desc(); - for (const auto &taskDesc : tasksDesc) { - v.emplace_back(nebula::Row({ - taskDesc.get_task_id(), - taskDesc.get_host().host, - apache::thrift::util::enumNameSafe(taskDesc.get_status()), - time::TimeConversion::unixSecondsToDateTime(taskDesc.get_start_time()), - time::TimeConversion::unixSecondsToDateTime(taskDesc.get_stop_time()), - })); - } - return finish(std::move(v)); - } - case meta::cpp2::AdminJobOp::SHOW_All: { - nebula::DataSet v({"Job Id", "Command", "Status", "Start Time", "Stop Time"}); - DCHECK(resp.value().job_desc_ref().has_value()); - if (!resp.value().job_desc_ref().has_value()) { - return Status::Error("Response unexpected"); - } - const auto &jobsDesc = *resp.value().job_desc_ref(); - for (const auto &jobDesc : jobsDesc) { - v.emplace_back(nebula::Row({ - jobDesc.get_id(), - apache::thrift::util::enumNameSafe(jobDesc.get_cmd()), - apache::thrift::util::enumNameSafe(jobDesc.get_status()), - time::TimeConversion::unixSecondsToDateTime(jobDesc.get_start_time()), - time::TimeConversion::unixSecondsToDateTime(jobDesc.get_stop_time()), - })); - } - return finish(std::move(v)); - } - case meta::cpp2::AdminJobOp::STOP: { - nebula::DataSet v({"Result"}); - v.emplace_back(nebula::Row({"Job stopped"})); - return finish(std::move(v)); - } - // no default so the compiler will warning when lack - } - DLOG(FATAL) << "Unknown job operation " << static_cast(jobOp); - return Status::Error("Unknown job job operation %d.", static_cast(jobOp)); + auto status = buildResult(jobOp, std::move(resp).value()); + NG_RETURN_IF_ERROR(status); + return finish(std::move(status).value()); }); } +StatusOr SubmitJobExecutor::buildResult(meta::cpp2::AdminJobOp jobOp, + meta::cpp2::AdminJobResult &&resp) { + switch (jobOp) { + case meta::cpp2::AdminJobOp::ADD: { + nebula::DataSet v({"New Job Id"}); + DCHECK(resp.job_id_ref().has_value()); + if (!resp.job_id_ref().has_value()) { + return Status::Error("Response unexpected."); + } + v.emplace_back(nebula::Row({*resp.job_id_ref()})); + return v; + } + case meta::cpp2::AdminJobOp::RECOVER: { + nebula::DataSet v({"Recovered job num"}); + DCHECK(resp.recovered_job_num_ref().has_value()); + if (!resp.recovered_job_num_ref().has_value()) { + return Status::Error("Response unexpected."); + } + v.emplace_back(nebula::Row({*resp.recovered_job_num_ref()})); + return v; + } + case meta::cpp2::AdminJobOp::SHOW: { + nebula::DataSet v({"Job Id(TaskId)", "Command(Dest)", "Status", "Start Time", "Stop Time"}); + DCHECK(resp.job_desc_ref().has_value()); + if (!resp.job_desc_ref().has_value()) { + return Status::Error("Response unexpected."); + } + DCHECK(resp.task_desc_ref().has_value()); + if (!resp.task_desc_ref().has_value()) { + return Status::Error("Response unexpected"); + } + auto &jobDesc = *resp.job_desc_ref(); + // job desc + v.emplace_back(nebula::Row({ + jobDesc.front().get_id(), + apache::thrift::util::enumNameSafe(jobDesc.front().get_cmd()), + apache::thrift::util::enumNameSafe(jobDesc.front().get_status()), + convertJobTimestampToDateTime(jobDesc.front().get_start_time()), + convertJobTimestampToDateTime(jobDesc.front().get_stop_time()), + })); + // tasks desc + auto &tasksDesc = *resp.get_task_desc(); + for (const auto &taskDesc : tasksDesc) { + v.emplace_back(nebula::Row({ + taskDesc.get_task_id(), + taskDesc.get_host().host, + apache::thrift::util::enumNameSafe(taskDesc.get_status()), + convertJobTimestampToDateTime(taskDesc.get_start_time()), + convertJobTimestampToDateTime(taskDesc.get_stop_time()), + })); + } + return v; + } + case meta::cpp2::AdminJobOp::SHOW_All: { + nebula::DataSet v({"Job Id", "Command", "Status", "Start Time", "Stop Time"}); + DCHECK(resp.job_desc_ref().has_value()); + if (!resp.job_desc_ref().has_value()) { + return Status::Error("Response unexpected"); + } + const auto &jobsDesc = *resp.job_desc_ref(); + for (const auto &jobDesc : jobsDesc) { + v.emplace_back(nebula::Row({ + jobDesc.get_id(), + apache::thrift::util::enumNameSafe(jobDesc.get_cmd()), + apache::thrift::util::enumNameSafe(jobDesc.get_status()), + convertJobTimestampToDateTime(jobDesc.get_start_time()), + convertJobTimestampToDateTime(jobDesc.get_stop_time()), + })); + } + return v; + } + case meta::cpp2::AdminJobOp::STOP: { + nebula::DataSet v({"Result"}); + v.emplace_back(nebula::Row({"Job stopped"})); + return v; + } + // no default so the compiler will warning when lack + } + DLOG(FATAL) << "Unknown job operation " << static_cast(jobOp); + return Status::Error("Unknown job job operation %d.", static_cast(jobOp)); +} + +Value SubmitJobExecutor::convertJobTimestampToDateTime(int64_t timestamp) { + return timestamp > 0 ? Value(time::TimeConversion::unixSecondsToDateTime(timestamp)) + : Value::kEmpty; +} } // namespace graph } // namespace nebula diff --git a/src/graph/executor/admin/SubmitJobExecutor.h b/src/graph/executor/admin/SubmitJobExecutor.h index dc8da2e75bf..08f777f2629 100644 --- a/src/graph/executor/admin/SubmitJobExecutor.h +++ b/src/graph/executor/admin/SubmitJobExecutor.h @@ -18,6 +18,11 @@ class SubmitJobExecutor final : public Executor { : Executor("SubmitJobExecutor", node, qctx) {} folly::Future execute() override; + + private: + FRIEND_TEST(JobTest, JobFinishTime); + StatusOr buildResult(meta::cpp2::AdminJobOp jobOp, meta::cpp2::AdminJobResult &&resp); + Value convertJobTimestampToDateTime(int64_t timestamp); }; } // namespace graph diff --git a/src/graph/executor/test/CMakeLists.txt b/src/graph/executor/test/CMakeLists.txt index 030f6b43dd9..c0a8925a452 100644 --- a/src/graph/executor/test/CMakeLists.txt +++ b/src/graph/executor/test/CMakeLists.txt @@ -81,6 +81,7 @@ nebula_add_test( CartesianProductTest.cpp AssignTest.cpp ShowQueriesTest.cpp + JobTest.cpp OBJECTS ${EXEC_QUERY_TEST_OBJS} LIBRARIES diff --git a/src/graph/executor/test/JobTest.cpp b/src/graph/executor/test/JobTest.cpp new file mode 100644 index 00000000000..58e4c33f79b --- /dev/null +++ b/src/graph/executor/test/JobTest.cpp @@ -0,0 +1,69 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include + +#include "common/time/TimeUtils.h" +#include "graph/context/QueryContext.h" +#include "graph/executor/admin/SubmitJobExecutor.h" +#include "graph/planner/plan/Admin.h" + +namespace nebula { +namespace graph { +class JobTest : public testing::Test {}; + +TEST_F(JobTest, JobFinishTime) { + { + meta::cpp2::AdminJobResult resp; + resp.set_job_id(0); + meta::cpp2::JobDesc jobDesc; + jobDesc.set_id(0); + jobDesc.set_start_time(123); + jobDesc.set_stop_time(0); + resp.set_job_desc({std::move(jobDesc)}); + meta::cpp2::TaskDesc taskDesc; + taskDesc.set_start_time(456); + taskDesc.set_stop_time(0); + resp.set_task_desc({std::move(taskDesc)}); + + auto qctx = std::make_unique(); + auto submitJob = SubmitJob::make( + qctx.get(), nullptr, meta::cpp2::AdminJobOp::SHOW, meta::cpp2::AdminCmd::UNKNOWN, {}); + auto submitJobExe = std::make_unique(submitJob, qctx.get()); + + auto status = submitJobExe->buildResult(meta::cpp2::AdminJobOp::SHOW, std::move(resp)); + EXPECT_TRUE(status.ok()); + auto result = std::move(status).value(); + EXPECT_EQ(result.rows.size(), 2); + EXPECT_EQ(result.rows[0][3], Value(time::TimeConversion::unixSecondsToDateTime(123))); + EXPECT_EQ(result.rows[0][4], Value::kEmpty); + EXPECT_EQ(result.rows[1][3], Value(time::TimeConversion::unixSecondsToDateTime(456))); + EXPECT_EQ(result.rows[1][4], Value::kEmpty); + } + { + meta::cpp2::AdminJobResult resp; + resp.set_job_id(0); + meta::cpp2::JobDesc jobDesc; + jobDesc.set_id(0); + jobDesc.set_start_time(123); + jobDesc.set_stop_time(0); + resp.set_job_desc({std::move(jobDesc)}); + + auto qctx = std::make_unique(); + auto submitJob = SubmitJob::make( + qctx.get(), nullptr, meta::cpp2::AdminJobOp::SHOW_All, meta::cpp2::AdminCmd::UNKNOWN, {}); + auto submitJobExe = std::make_unique(submitJob, qctx.get()); + + auto status = submitJobExe->buildResult(meta::cpp2::AdminJobOp::SHOW_All, std::move(resp)); + EXPECT_TRUE(status.ok()); + auto result = std::move(status).value(); + EXPECT_EQ(result.rows.size(), 1); + EXPECT_EQ(result.rows[0][3], Value(time::TimeConversion::unixSecondsToDateTime(123))); + EXPECT_EQ(result.rows[0][4], Value::kEmpty); + } +} +} // namespace graph +} // namespace nebula diff --git a/src/graph/optimizer/OptimizerUtils.cpp b/src/graph/optimizer/OptimizerUtils.cpp index 79ca8518634..53f7e89c240 100644 --- a/src/graph/optimizer/OptimizerUtils.cpp +++ b/src/graph/optimizer/OptimizerUtils.cpp @@ -922,7 +922,7 @@ void OptimizerUtils::copyIndexScanData(const nebula::graph::IndexScan* from, to->setDedup(from->dedup()); to->setOrderBy(from->orderBy()); to->setLimit(from->limit()); - to->setFilter(from->filter()); + to->setFilter(from->filter() == nullptr ? nullptr : from->filter()->clone()); } } // namespace graph diff --git a/src/graph/optimizer/rule/PushFilterDownGetNbrsRule.cpp b/src/graph/optimizer/rule/PushFilterDownGetNbrsRule.cpp index 3e74820e877..4ffa80807ae 100644 --- a/src/graph/optimizer/rule/PushFilterDownGetNbrsRule.cpp +++ b/src/graph/optimizer/rule/PushFilterDownGetNbrsRule.cpp @@ -74,11 +74,10 @@ StatusOr PushFilterDownGetNbrsRule::transform( newFilterGroupNode = OptGroupNode::create(ctx, newFilter, filterGroupNode->group()); } - auto newGNFilter = condition->encode(); - if (!gn->filter().empty()) { - auto filterExpr = Expression::decode(pool, gn->filter()); - auto logicExpr = LogicalExpression::makeAnd(pool, condition, filterExpr); - newGNFilter = logicExpr->encode(); + auto newGNFilter = condition; + if (gn->filter() != nullptr) { + auto logicExpr = LogicalExpression::makeAnd(pool, condition, gn->filter()->clone()); + newGNFilter = logicExpr; } auto newGN = static_cast(gn->clone()); diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp index 02a3eead3e6..69b8632248a 100644 --- a/src/graph/planner/plan/Query.cpp +++ b/src/graph/planner/plan/Query.cpp @@ -23,8 +23,7 @@ std::unique_ptr Explore::explain() const { addDescription("space", folly::to(space_), desc.get()); addDescription("dedup", util::toJson(dedup_), desc.get()); addDescription("limit", folly::to(limit_), desc.get()); - auto filter = - filter_.empty() ? filter_ : Expression::decode(qctx_->objPool(), filter_)->toString(); + std::string filter = filter_ == nullptr ? "" : filter_->toString(); addDescription("filter", filter, desc.get()); addDescription("orderBy", folly::toJson(util::toJson(orderBy_)), desc.get()); return desc; diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h index 92ba36dbcbc..315baa43840 100644 --- a/src/graph/planner/plan/Query.h +++ b/src/graph/planner/plan/Query.h @@ -37,7 +37,9 @@ class Explore : public SingleInputNode { int64_t limit() const { return limit_; } - const std::string& filter() const { return filter_; } + const Expression* filter() const { return filter_; } + + Expression* filter() { return filter_; } const std::vector& orderBy() const { return orderBy_; } @@ -45,7 +47,7 @@ class Explore : public SingleInputNode { void setLimit(int64_t limit) { limit_ = limit; } - void setFilter(std::string filter) { filter_ = std::move(filter); } + void setFilter(Expression* filter) { filter_ = filter; } void setOrderBy(std::vector orderBy) { orderBy_ = std::move(orderBy); } @@ -58,13 +60,13 @@ class Explore : public SingleInputNode { GraphSpaceID space, bool dedup, int64_t limit, - std::string filter, + Expression* filter, std::vector orderBy) : SingleInputNode(qctx, kind, input), space_(space), dedup_(dedup), limit_(limit), - filter_(std::move(filter)), + filter_(filter), orderBy_(std::move(orderBy)) {} Explore(QueryContext* qctx, Kind kind, PlanNode* input, GraphSpaceID space) @@ -76,7 +78,7 @@ class Explore : public SingleInputNode { GraphSpaceID space_; bool dedup_{false}; int64_t limit_{std::numeric_limits::max()}; - std::string filter_; + Expression* filter_{nullptr}; std::vector orderBy_; }; @@ -108,7 +110,7 @@ class GetNeighbors final : public Explore { bool random = false, std::vector orderBy = {}, int64_t limit = -1, - std::string filter = "") { + Expression* filter = nullptr) { auto gn = make(qctx, input, space); gn->setSrc(src); gn->setEdgeTypes(std::move(edgeTypes)); @@ -121,7 +123,7 @@ class GetNeighbors final : public Explore { gn->setDedup(dedup); gn->setOrderBy(std::move(orderBy)); gn->setLimit(limit); - gn->setFilter(std::move(filter)); + gn->setFilter(filter); return gn; } @@ -199,7 +201,7 @@ class GetVertices final : public Explore { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new GetVertices(qctx, input, space, @@ -209,7 +211,7 @@ class GetVertices final : public Explore { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } Expression* src() const { return src_; } @@ -237,15 +239,8 @@ class GetVertices final : public Explore { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) - : Explore(qctx, - Kind::kGetVertices, - input, - space, - dedup, - limit, - std::move(filter), - std::move(orderBy)), + Expression* filter) + : Explore(qctx, Kind::kGetVertices, input, space, dedup, limit, filter, std::move(orderBy)), src_(src), props_(std::move(props)), exprs_(std::move(exprs)) {} @@ -278,7 +273,7 @@ class GetEdges final : public Explore { bool dedup = false, int64_t limit = std::numeric_limits::max(), std::vector orderBy = {}, - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new GetEdges(qctx, input, space, @@ -291,7 +286,7 @@ class GetEdges final : public Explore { dedup, limit, std::move(orderBy), - std::move(filter))); + filter)); } Expression* src() const { return src_; } @@ -326,15 +321,8 @@ class GetEdges final : public Explore { bool dedup, int64_t limit, std::vector orderBy, - std::string filter) - : Explore(qctx, - Kind::kGetEdges, - input, - space, - dedup, - limit, - std::move(filter), - std::move(orderBy)), + Expression* filter) + : Explore(qctx, Kind::kGetEdges, input, space, dedup, limit, filter, std::move(orderBy)), src_(src), type_(type), ranking_(ranking), @@ -374,7 +362,7 @@ class IndexScan : public Explore { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new IndexScan(qctx, input, space, @@ -386,7 +374,7 @@ class IndexScan : public Explore { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } const std::vector& queryContext() const { return contexts_; } @@ -426,9 +414,9 @@ class IndexScan : public Explore { bool dedup, std::vector orderBy, int64_t limit, - std::string filter, + Expression* filter, Kind kind = Kind::kIndexScan) - : Explore(qctx, kind, input, space, dedup, limit, std::move(filter), std::move(orderBy)) { + : Explore(qctx, kind, input, space, dedup, limit, filter, std::move(orderBy)) { contexts_ = std::move(contexts); returnCols_ = std::move(returnCols); isEdge_ = isEdge; diff --git a/src/graph/planner/plan/Scan.h b/src/graph/planner/plan/Scan.h index 03040d62012..4df13613d61 100644 --- a/src/graph/planner/plan/Scan.h +++ b/src/graph/planner/plan/Scan.h @@ -29,7 +29,7 @@ class EdgeIndexScan : public IndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter, + Expression* filter, Kind kind) : IndexScan(qctx, input, @@ -42,7 +42,7 @@ class EdgeIndexScan : public IndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, kind), edgeType_(edgeType) {} @@ -62,7 +62,7 @@ class EdgeIndexPrefixScan : public EdgeIndexScan { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new EdgeIndexPrefixScan(qctx, input, edgeType, @@ -74,7 +74,7 @@ class EdgeIndexPrefixScan : public EdgeIndexScan { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } private: @@ -89,7 +89,7 @@ class EdgeIndexPrefixScan : public EdgeIndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) + Expression* filter) : EdgeIndexScan(qctx, input, edgeType, @@ -101,7 +101,7 @@ class EdgeIndexPrefixScan : public EdgeIndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, Kind::kEdgeIndexPrefixScan) {} }; @@ -118,7 +118,7 @@ class EdgeIndexRangeScan : public EdgeIndexScan { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new EdgeIndexRangeScan(qctx, input, edgeType, @@ -130,7 +130,7 @@ class EdgeIndexRangeScan : public EdgeIndexScan { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } private: @@ -145,7 +145,7 @@ class EdgeIndexRangeScan : public EdgeIndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) + Expression* filter) : EdgeIndexScan(qctx, input, edgeType, @@ -157,7 +157,7 @@ class EdgeIndexRangeScan : public EdgeIndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, Kind::kEdgeIndexRangeScan) {} }; @@ -174,7 +174,7 @@ class EdgeIndexFullScan final : public EdgeIndexScan { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new EdgeIndexFullScan(qctx, input, edgeType, @@ -186,7 +186,7 @@ class EdgeIndexFullScan final : public EdgeIndexScan { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } private: @@ -201,7 +201,7 @@ class EdgeIndexFullScan final : public EdgeIndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) + Expression* filter) : EdgeIndexScan(qctx, input, edgeType, @@ -213,7 +213,7 @@ class EdgeIndexFullScan final : public EdgeIndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, Kind::kEdgeIndexFullScan) {} }; @@ -235,7 +235,7 @@ class TagIndexScan : public IndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter, + Expression* filter, Kind kind) : IndexScan(qctx, input, @@ -248,7 +248,7 @@ class TagIndexScan : public IndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, kind), tagName_(tagName) {} @@ -268,7 +268,7 @@ class TagIndexPrefixScan : public TagIndexScan { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new TagIndexPrefixScan(qctx, input, tagName, @@ -280,7 +280,7 @@ class TagIndexPrefixScan : public TagIndexScan { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } private: @@ -295,7 +295,7 @@ class TagIndexPrefixScan : public TagIndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) + Expression* filter) : TagIndexScan(qctx, input, tagName, @@ -307,7 +307,7 @@ class TagIndexPrefixScan : public TagIndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, Kind::kTagIndexPrefixScan) {} }; @@ -324,7 +324,7 @@ class TagIndexRangeScan : public TagIndexScan { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new TagIndexRangeScan(qctx, input, tagName, @@ -336,7 +336,7 @@ class TagIndexRangeScan : public TagIndexScan { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } private: @@ -351,7 +351,7 @@ class TagIndexRangeScan : public TagIndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) + Expression* filter) : TagIndexScan(qctx, input, tagName, @@ -363,7 +363,7 @@ class TagIndexRangeScan : public TagIndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, Kind::kTagIndexRangeScan) {} }; @@ -380,7 +380,7 @@ class TagIndexFullScan final : public TagIndexScan { bool dedup = false, std::vector orderBy = {}, int64_t limit = std::numeric_limits::max(), - std::string filter = "") { + Expression* filter = nullptr) { return qctx->objPool()->add(new TagIndexFullScan(qctx, input, tagName, @@ -392,7 +392,7 @@ class TagIndexFullScan final : public TagIndexScan { dedup, std::move(orderBy), limit, - std::move(filter))); + filter)); } private: @@ -407,7 +407,7 @@ class TagIndexFullScan final : public TagIndexScan { bool dedup, std::vector orderBy, int64_t limit, - std::string filter) + Expression* filter) : TagIndexScan(qctx, input, tagName, @@ -419,7 +419,7 @@ class TagIndexFullScan final : public TagIndexScan { dedup, std::move(orderBy), limit, - std::move(filter), + filter, Kind::kTagIndexFullScan) {} }; diff --git a/src/graph/service/GraphService.cpp b/src/graph/service/GraphService.cpp index 6b217fb0c1f..5bed12c6187 100644 --- a/src/graph/service/GraphService.cpp +++ b/src/graph/service/GraphService.cpp @@ -153,6 +153,13 @@ folly::Future GraphService::future_execute(int64_t sessionId, return future; } +folly::Future GraphService::future_executeJson(int64_t sessionId, + const std::string& query) { + auto rawResp = future_execute(sessionId, query).get(); + auto respJsonObj = rawResp.toJson(); + return folly::toJson(respJsonObj); +} + bool GraphService::auth(const std::string& username, const std::string& password) { if (!FLAGS_enable_authorize) { return true; diff --git a/src/graph/service/GraphService.h b/src/graph/service/GraphService.h index 0e15ef0e873..f72c36bb39e 100644 --- a/src/graph/service/GraphService.h +++ b/src/graph/service/GraphService.h @@ -36,6 +36,9 @@ class GraphService final : public cpp2::GraphServiceSvIf { folly::Future future_execute(int64_t sessionId, const std::string& stmt) override; + folly::Future future_executeJson(int64_t sessionId, + const std::string& stmt) override; + private: bool auth(const std::string& username, const std::string& password); diff --git a/src/graph/validator/FetchEdgesValidator.cpp b/src/graph/validator/FetchEdgesValidator.cpp index 5909e1bbc4a..60c5636389f 100644 --- a/src/graph/validator/FetchEdgesValidator.cpp +++ b/src/graph/validator/FetchEdgesValidator.cpp @@ -46,7 +46,7 @@ Status FetchEdgesValidator::toPlan() { dedup_, limit_, std::move(orderBy_), - std::move(filter_)); + filter_); getEdgesNode->setInputVar(edgeKeysVar); getEdgesNode->setColNames(geColNames_); // the pipe will set the input variable diff --git a/src/graph/validator/FetchEdgesValidator.h b/src/graph/validator/FetchEdgesValidator.h index 6009b12e7e7..c20e94ca3d0 100644 --- a/src/graph/validator/FetchEdgesValidator.h +++ b/src/graph/validator/FetchEdgesValidator.h @@ -72,7 +72,7 @@ class FetchEdgesValidator final : public Validator { bool dedup_{false}; int64_t limit_{std::numeric_limits::max()}; std::vector orderBy_{}; - std::string filter_{""}; + Expression* filter_{nullptr}; // valid when yield expression not require storage // So expression like these will be evaluate in Project Executor bool withYield_{false}; diff --git a/src/graph/validator/FetchVerticesValidator.cpp b/src/graph/validator/FetchVerticesValidator.cpp index fc72408d85c..10f870ea231 100644 --- a/src/graph/validator/FetchVerticesValidator.cpp +++ b/src/graph/validator/FetchVerticesValidator.cpp @@ -37,7 +37,7 @@ Status FetchVerticesValidator::toPlan() { dedup_, std::move(orderBy_), limit_, - std::move(filter_)); + filter_); getVerticesNode->setInputVar(vidsVar); getVerticesNode->setColNames(gvColNames_); // pipe will set the input variable diff --git a/src/graph/validator/FetchVerticesValidator.h b/src/graph/validator/FetchVerticesValidator.h index 4a9dd4a1506..b915a3cc0de 100644 --- a/src/graph/validator/FetchVerticesValidator.h +++ b/src/graph/validator/FetchVerticesValidator.h @@ -52,7 +52,7 @@ class FetchVerticesValidator final : public Validator { bool dedup_{false}; std::vector orderBy_{}; int64_t limit_{std::numeric_limits::max()}; - std::string filter_{}; + Expression* filter_{nullptr}; // valid when yield expression not require storage // So expression like these will be evaluate in Project Executor bool withYield_{false}; diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp index a99edceadb5..d913e3cbad6 100644 --- a/src/graph/validator/MutateValidator.cpp +++ b/src/graph/validator/MutateValidator.cpp @@ -148,7 +148,7 @@ Status InsertEdgesValidator::toPlan() { nullptr, spaceId_, std::move(edges_), - std::move(propNames_), + std::move(entirePropNames_), ifNotExists_, useChainInsert); root_ = doNode; @@ -196,6 +196,11 @@ Status InsertEdgesValidator::prepareEdges() { auto useToss = isoLevel == IsoLevel::TOSS; auto size = useToss ? rows_.size() : rows_.size() * 2; edges_.reserve(size); + + size_t fieldNum = schema_->getNumFields(); + for (size_t j = 0; j < fieldNum; ++j) { + entirePropNames_.emplace_back(schema_->field(j)->name()); + } for (auto i = 0u; i < rows_.size(); i++) { auto *row = rows_[i]; if (propNames_.size() != row->values().size()) { @@ -233,6 +238,34 @@ Status InsertEdgesValidator::prepareEdges() { auto valsRet = SchemaUtil::toValueVec(row->values()); NG_RETURN_IF_ERROR(valsRet); auto props = std::move(valsRet).value(); + + std::vector entirePropValues; + for (size_t j = 0; j < fieldNum; ++j) { + auto *field = schema_->field(j); + auto propName = entirePropNames_[j]; + auto iter = std::find(propNames_.begin(), propNames_.end(), propName); + if (iter == propNames_.end()) { + if (field->hasDefault()) { + auto *defaultValue = field->defaultValue(); + DCHECK(!!defaultValue); + auto v = defaultValue->eval(QueryExpressionContext()(nullptr)); + entirePropValues.emplace_back(v); + } else { + if (!field->nullable()) { + return Status::SemanticError( + "The property `%s' is not nullable and has no default value.", field->name()); + } + entirePropValues.emplace_back(Value(NullType::__NULL__)); + } + } else { + auto v = props[std::distance(propNames_.begin(), iter)]; + if (!field->nullable() && v.isNull()) { + return Status::SemanticError("The non-nullable property `%s' could not be NULL.", + field->name()); + } + entirePropValues.emplace_back(v); + } + } storage::cpp2::NewEdge edge; storage::cpp2::EdgeKey key; @@ -241,7 +274,7 @@ Status InsertEdgesValidator::prepareEdges() { key.set_edge_type(edgeType_); key.set_ranking(rank); edge.set_key(key); - edge.set_props(std::move(props)); + edge.set_props(std::move(entirePropValues)); edges_.emplace_back(edge); if (!useToss) { // inbound diff --git a/src/graph/validator/MutateValidator.h b/src/graph/validator/MutateValidator.h index 8ebd2a876df..58464c32b74 100644 --- a/src/graph/validator/MutateValidator.h +++ b/src/graph/validator/MutateValidator.h @@ -58,6 +58,7 @@ class InsertEdgesValidator final : public Validator { EdgeType edgeType_{-1}; std::shared_ptr schema_; std::vector propNames_; + std::vector entirePropNames_; std::vector rows_; std::vector edges_; }; diff --git a/src/graph/validator/UseValidator.cpp b/src/graph/validator/UseValidator.cpp index c80ded6f201..7049bb752d5 100644 --- a/src/graph/validator/UseValidator.cpp +++ b/src/graph/validator/UseValidator.cpp @@ -43,12 +43,9 @@ Status UseValidator::validateImpl() { Status UseValidator::toPlan() { // The input will be set by father validator later. - auto switchSpace = SwitchSpace::make(qctx_, nullptr, *spaceName_); - qctx_->rctx()->session()->updateSpaceName(*spaceName_); - auto session = qctx_->rctx()->session()->getSession(); - auto update = UpdateSession::make(qctx_, switchSpace, std::move(session)); - root_ = update; - tail_ = switchSpace; + auto reg = SwitchSpace::make(qctx_, nullptr, *spaceName_); + root_ = reg; + tail_ = root_; return Status::OK(); } } // namespace graph diff --git a/src/graph/validator/test/AdminValidatorTest.cpp b/src/graph/validator/test/AdminValidatorTest.cpp index 19c62c50522..4bc3f737cea 100644 --- a/src/graph/validator/test/AdminValidatorTest.cpp +++ b/src/graph/validator/test/AdminValidatorTest.cpp @@ -18,8 +18,7 @@ TEST_F(AdminValidatorTest, SpaceTest) { checkResult("CREATE SPACE TEST(vid_type = fixed_string(2)); DESC SPACE TEST;", expected)); } { - std::vector expected = { - PK::kUpdateSession, PK::kSwitchSpace, PK::kCreateSpace, PK::kStart}; + std::vector expected = {PK::kSwitchSpace, PK::kCreateSpace, PK::kStart}; ASSERT_TRUE(checkResult("CREATE SPACE TEST(vid_type = fixed_string(2)); USE TEST;", expected)); } } diff --git a/src/graph/validator/test/MutateValidatorTest.cpp b/src/graph/validator/test/MutateValidatorTest.cpp index b60a4562b05..a0677381f4b 100644 --- a/src/graph/validator/test/MutateValidatorTest.cpp +++ b/src/graph/validator/test/MutateValidatorTest.cpp @@ -44,11 +44,18 @@ TEST_F(MutateValidatorTest, InsertEdgeTest) { ASSERT_FALSE(checkResult(cmd, {})); } // vid use function call + { + auto cmd = + "INSERT EDGE like(start, end, likeness) VALUES lower(\"Lily\")->\"Tom\":(2010, " + "2020, 90);"; + ASSERT_TRUE(checkResult(cmd, {PK::kInsertEdges, PK::kStart})); + } + // vid use function call { auto cmd = "INSERT EDGE like(start, end) VALUES lower(\"Lily\")->\"Tom\":(2010, " "2020);"; - ASSERT_TRUE(checkResult(cmd, {PK::kInsertEdges, PK::kStart})); + ASSERT_FALSE(checkResult(cmd, {PK::kInsertEdges, PK::kStart})); } } diff --git a/src/meta/http/MetaHttpDownloadHandler.cpp b/src/meta/http/MetaHttpDownloadHandler.cpp index 230637efb80..1912728333e 100644 --- a/src/meta/http/MetaHttpDownloadHandler.cpp +++ b/src/meta/http/MetaHttpDownloadHandler.cpp @@ -21,6 +21,8 @@ #include "webservice/Common.h" #include "webservice/WebService.h" +DECLARE_int32(ws_storage_http_port); + namespace nebula { namespace meta { diff --git a/src/meta/http/MetaHttpIngestHandler.cpp b/src/meta/http/MetaHttpIngestHandler.cpp index 85f1f0b7240..0f73c47f5a4 100644 --- a/src/meta/http/MetaHttpIngestHandler.cpp +++ b/src/meta/http/MetaHttpIngestHandler.cpp @@ -18,6 +18,9 @@ #include "webservice/Common.h" #include "webservice/WebService.h" +DECLARE_int32(ws_storage_http_port); +DECLARE_int32(ws_storage_h2_port); + DEFINE_int32(meta_ingest_thread_num, 3, "Meta daemon's ingest thread number"); namespace nebula { diff --git a/src/storage/exec/GetNeighborsNode.h b/src/storage/exec/GetNeighborsNode.h index b3f0eab036f..96c4557810a 100644 --- a/src/storage/exec/GetNeighborsNode.h +++ b/src/storage/exec/GetNeighborsNode.h @@ -160,7 +160,7 @@ class GetNeighborsSampleNode : public GetNeighborsNode { } RowReaderWrapper reader; - auto samples = std::move(*sampler_).samples(); + auto samples = sampler_->samples(); for (auto& sample : samples) { auto columnIdx = std::get<4>(sample); // add edge prop value to the target column diff --git a/src/storage/test/StorageServiceHandlerTest.cpp b/src/storage/test/StorageServiceHandlerTest.cpp deleted file mode 100644 index 385e987549d..00000000000 --- a/src/storage/test/StorageServiceHandlerTest.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2018 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include - -#include "common/base/Base.h" -#include "common/fs/TempDir.h" -#include "common/utils/NebulaKeyUtils.h" -#include "storage/StorageServiceHandler.h" -#include "storage/mutate/AddVerticesProcessor.h" -#include "storage/test/TestUtils.h" - -namespace nebula { -namespace storage { - -TEST(StorageServiceHandlerTest, FutureAddVerticesTest) { - fs::TempDir rootPath("/tmp/FutureAddVerticesTest.XXXXXX"); - cpp2::AddVerticesRequest req; - req.set_space_id(0); - req.overwritable = true; - - LOG(INFO) << "Build FutureAddVerticesTest..."; - req.parts.emplace(0, TestUtils::setupVertices(0, 0, 10, 0, 10)); - req.parts.emplace(1, TestUtils::setupVertices(1, 0, 20, 0, 30)); - LOG(INFO) << "Test FutureAddVerticesTest..."; - std::unique_ptr kvstore = TestUtils::initKV(rootPath.path()); - auto schemaMan = TestUtils::mockSchemaMan(); - auto indexMan = TestUtils::mockIndexMan(); - auto storageServiceHandler = std::make_unique( - kvstore.get(), schemaMan.get(), indexMan.get(), nullptr); - auto resp = storageServiceHandler->future_addVertices(req).get(); - EXPECT_EQ(typeid(nebula::cpp2::ExecResponse).name(), typeid(resp).name()); - - LOG(INFO) << "Check ErrorCode of AddVerticesProcessor..."; - ASSERT_EQ(0, resp.result.failed_codes.size()); - - LOG(INFO) << "Verify the vertices data..."; - auto prefix = NebulaKeyUtils::vertexPrefix(1, 19); - std::unique_ptr iter; - ASSERT_EQ(cpp2::ErrorCode::SUCCEEDED, kvstore->prefix(0, 1, prefix, &iter)); - TagID tagId = 0; - while (iter->valid()) { - ASSERT_EQ(TestUtils::encodeValue(1, 19, tagId), iter->val()); - tagId++; - iter->next(); - } - ASSERT_EQ(30, tagId); - LOG(INFO) << "Test FutureAddVerticesTest..."; -} -} // namespace storage -} // namespace nebula - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - folly::init(&argc, &argv, true); - google::SetStderrLogging(google::INFO); - return RUN_ALL_TESTS(); -} diff --git a/src/webservice/CMakeLists.txt b/src/webservice/CMakeLists.txt index c0b6f84b405..a13bad1ea52 100644 --- a/src/webservice/CMakeLists.txt +++ b/src/webservice/CMakeLists.txt @@ -10,9 +10,10 @@ nebula_add_library( GetFlagsHandler.cpp SetFlagsHandler.cpp GetStatsHandler.cpp - Router.cpp - StatusHandler.cpp + Router.cpp + StatusHandler.cpp ) + set_target_properties(ws_obj PROPERTIES COMPILE_FLAGS "-Wno-error=format-security") nebula_add_library( diff --git a/src/webservice/Common.cpp b/src/webservice/Common.cpp index b89955a9e73..034b836e57d 100644 --- a/src/webservice/Common.cpp +++ b/src/webservice/Common.cpp @@ -6,7 +6,21 @@ #include "webservice/Common.h" +#include + DEFINE_int32(ws_meta_http_port, 11000, "Port to listen on Meta with HTTP protocol"); DEFINE_int32(ws_meta_h2_port, 11002, "Port to listen on Meta with HTTP/2 protocol"); DEFINE_int32(ws_storage_http_port, 12000, "Port to listen on Storage with HTTP protocol"); DEFINE_int32(ws_storage_h2_port, 12002, "Port to listen on Storage with HTTP/2 protocol"); + +namespace nebula { + +std::unordered_map WebServiceUtils::kStatusStringMap_ = { + {HttpStatusCode::OK, "OK"}, + {HttpStatusCode::BAD_REQUEST, "Bad Request"}, + {HttpStatusCode::FORBIDDEN, "Forbidden"}, + {HttpStatusCode::NOT_FOUND, "Not Found"}, + {HttpStatusCode::METHOD_NOT_ALLOWED, "Method Not Allowed"}, +}; + +} // namespace nebula diff --git a/src/webservice/Common.h b/src/webservice/Common.h index 6b0a06bc704..e782724b67d 100644 --- a/src/webservice/Common.h +++ b/src/webservice/Common.h @@ -7,12 +7,9 @@ #ifndef WEBSERVICE_COMMON_H_ #define WEBSERVICE_COMMON_H_ -#include "common/base/Base.h" - -DECLARE_int32(ws_meta_http_port); -DECLARE_int32(ws_meta_h2_port); -DECLARE_int32(ws_storage_http_port); -DECLARE_int32(ws_storage_h2_port); +#include +#include +#include namespace nebula { @@ -23,7 +20,7 @@ enum class HttpCode { E_ILLEGAL_ARGUMENT = -3, }; -enum class HttpStatusCode { +enum class HttpStatusCode : int32_t { OK = 200, BAD_REQUEST = 400, FORBIDDEN = 403, @@ -31,19 +28,15 @@ enum class HttpStatusCode { METHOD_NOT_ALLOWED = 405, }; -static std::map statusStringMap{ - {HttpStatusCode::OK, "OK"}, - {HttpStatusCode::BAD_REQUEST, "Bad Request"}, - {HttpStatusCode::FORBIDDEN, "Forbidden"}, - {HttpStatusCode::NOT_FOUND, "Not Found"}, - {HttpStatusCode::METHOD_NOT_ALLOWED, "Method Not Allowed"}}; - class WebServiceUtils final { public: static int32_t to(HttpStatusCode code) { return static_cast(code); } + static std::string toString(HttpStatusCode code) { return kStatusStringMap_[code]; } - static std::string toString(HttpStatusCode code) { return statusStringMap[code]; } + private: + static std::unordered_map kStatusStringMap_; }; } // namespace nebula + #endif // WEBSERVICE_COMMON_H_ diff --git a/src/webservice/test/CMakeLists.txt b/src/webservice/test/CMakeLists.txt index feb17e1023e..f1a01457c3f 100644 --- a/src/webservice/test/CMakeLists.txt +++ b/src/webservice/test/CMakeLists.txt @@ -11,6 +11,7 @@ nebula_add_test( OBJECTS $ $ + $ $ $ $ @@ -31,6 +32,7 @@ nebula_add_test( OBJECTS $ $ + $ $ $ $ diff --git a/tests/Makefile b/tests/Makefile index 10fc66d046b..36963cbf2c7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -61,16 +61,16 @@ sess: currdir python3 -m pytest -m "not skip" -k "not tck" job/test_session.py jobs: currdir - python3 -m pytest -m "not skip" -k "not tck" job/test_jobs.py + python3 -m pytest -m "not skip" tck/steps/test_jobs.py -test: sess jobs +test: sess python3 -m pytest -n$(J) --dist=loadfile -m "not skip" -k "not tck" $(TEST_DIR) slow-query: currdir python3 -m pytest -n$(J) -m "not skip" tck/steps/test_kill_slow_query_via_same_service.py && \ python3 -m pytest -n$(J) -m "not skip" tck/steps/test_kill_slow_query_via_different_service.py -tck: slow-query +tck: jobs slow-query python3 -m pytest -n$(J) -m "not skip" tck/steps/test_tck.py fail: currdir diff --git a/tests/common/comparator.py b/tests/common/comparator.py index 3468f7616f4..02995719e59 100644 --- a/tests/common/comparator.py +++ b/tests/common/comparator.py @@ -29,11 +29,13 @@ def __init__(self, strict=True, order=False, contains=CmpType.EQUAL, + first_n_records=-1, decode_type='utf-8', vid_fn=None): self._strict = strict self._order = order self._contains = contains + self._first_n_records=first_n_records self._decode_type = decode_type self._vid_fn = vid_fn @@ -65,13 +67,16 @@ def compare(self, resp: DataSet, expect: DataSet): if ln != self.bstr(rn): return False, -2 if self._order: - for i in range(0, len(expect.rows)): - cmp = self.compare_row(resp.rows[i], expect.rows[i]) - if self._whether_return(cmp): - return False, i - if self._contains == CmpType.CONTAINS: + if self._contains == CmpType.CONTAINS and self._first_n_records < 0: + for i in range(0, len(expect.rows)): + cmp = self.compare_row(resp.rows[i], expect.rows[i]) + if self._whether_return(cmp): + return False, i return True, None - return len(resp.rows) == len(expect.rows), -1 + elif self._contains == CmpType.CONTAINS and self._first_n_records > 0: + return self._compare_list(resp.rows[0:self._first_n_records], expect.rows, self.compare_row) + else: + return len(resp.rows) == len(expect.rows), -1 return self._compare_list(resp.rows, expect.rows, self.compare_row, self._contains) diff --git a/tests/common/utils.py b/tests/common/utils.py index cb08622d45b..722298e10b7 100644 --- a/tests/common/utils.py +++ b/tests/common/utils.py @@ -426,7 +426,7 @@ def load_csv_data( for line in schemas.splitlines(): resp_ok(sess, line.strip(), True) - # wait heartbeat_interval_secs seconds for schema synchronization + # wait heartbeat_interval_secs + 1 seconds for schema synchronization time.sleep(2) for fd in config["files"]: diff --git a/tests/job/test_jobs.py b/tests/job/test_jobs.py deleted file mode 100644 index ace541646bb..00000000000 --- a/tests/job/test_jobs.py +++ /dev/null @@ -1,89 +0,0 @@ -# --coding:utf-8-- -# -# Copyright (c) 2020 vesoft inc. All rights reserved. -# -# This source code is licensed under Apache 2.0 License, -# attached with Common Clause Condition 1.0, found in the LICENSES directory. - -import re -import time - -from nebula2.common import ttypes -from tests.common.nebula_test_suite import NebulaTestSuite - -class TestJobs(NebulaTestSuite): - def test_failed(self): - # submit without space - resp = self.client.execute('SUBMIT JOB COMPACT;') - self.check_resp_failed(resp, ttypes.ErrorCode.E_SEMANTIC_ERROR) - # show one not exists - resp = self.client.execute('SHOW JOB 233;') - self.check_resp_failed(resp, ttypes.ErrorCode.E_EXECUTION_ERROR) - # stop one not exists - resp = self.client.execute('STOP JOB 233;') - self.check_resp_failed(resp, ttypes.ErrorCode.E_EXECUTION_ERROR) - - def test_succeeded(self): - def check_jobs_resp_obj(resp_row, job_name): - assert resp_row[1].as_string() == job_name - assert resp_row[2].is_string() - assert resp_row[3].is_datetime() - assert resp_row[4].is_datetime() - - resp = self.client.execute('CREATE SPACE IF NOT EXISTS space_for_jobs(partition_num=9, replica_factor=1, vid_type=FIXED_STRING(20));' - 'USE space_for_jobs;') - self.check_resp_succeeded(resp) - - resp = self.client.execute('SUBMIT JOB COMPACT;') - self.check_resp_succeeded(resp) - expect_col_names = ['New Job Id'] - self.check_column_names(resp, expect_col_names) - expect_values = [[re.compile(r'\d+')]] - self.check_result(resp, expect_values, is_regex=True) - time.sleep(1) - - resp = self.client.execute('SUBMIT JOB FLUSH;') - self.check_resp_succeeded(resp) - expect_col_names = ['New Job Id'] - self.check_column_names(resp, expect_col_names) - expect_values = [[re.compile(r'\d+')]] - self.check_result(resp, expect_values, is_regex=True) - time.sleep(1) - - resp = self.client.execute('SUBMIT JOB STATS;') - self.check_resp_succeeded(resp) - expect_col_names = ['New Job Id'] - self.check_column_names(resp, expect_col_names) - expect_values = [[re.compile(r'\d+')]] - self.check_result(resp, expect_values, is_regex=True) - - time.sleep(10) - resp = self.client.execute('SHOW JOBS;') - self.check_resp_succeeded(resp) - expect_col_names = ['Job Id', 'Command', 'Status', 'Start Time', 'Stop Time'] - self.check_column_names(resp, expect_col_names) - check_jobs_resp_obj(resp.row_values(0), 'STATS') - check_jobs_resp_obj(resp.row_values(1), 'FLUSH') - check_jobs_resp_obj(resp.row_values(2), 'COMPACT') - - job_id = resp.row_values(0)[0].as_int() - resp = self.client.execute('SHOW JOB {};'.format(job_id)) - self.check_resp_succeeded(resp) - expect_col_names = ['Job Id(TaskId)', 'Command(Dest)', 'Status', 'Start Time', 'Stop Time'] - check_jobs_resp_obj(resp.row_values(0), 'STATS') - - job_id = resp.row_values(0)[0].as_int() - stop_job_resp = self.client.execute('STOP JOB {};'.format(job_id)) - if resp.row_values(0)[2].as_string() == "FINISHED": - # Executin error if the job is finished - self.check_resp_failed(stop_job_resp, ttypes.ErrorCode.E_EXECUTION_ERROR) - else: - self.check_resp_succeeded(stop_job_resp) - - # This is skipped becuase it is hard to simulate the situation - # resp = self.client.execute('RECOVER JOB;') - # self.check_resp_succeeded(resp) - # expect_col_names = ['Recovered job num'] - # self.check_column_names(resp, expect_col_names) - # expect_values = [[0]] - # self.check_result(resp, expect_values) diff --git a/tests/job/test_session.py b/tests/job/test_session.py index 6f8e9873cb0..97178cc2e7c 100644 --- a/tests/job/test_session.py +++ b/tests/job/test_session.py @@ -88,9 +88,9 @@ def test_sessions(self): for row in resp.rows(): if bytes.decode(row.values[1].get_sVal()) == 'session_user': session_id = row.values[0].get_iVal() - assert row.values[2].get_sVal() == b'' - assert row.values[3].getType() == ttypes.Value.DTVAL - assert row.values[4].getType() == ttypes.Value.DTVAL + assert row.values[2].get_sVal() == b'', f"resp: {resp}" + assert row.values[3].getType() == ttypes.Value.DTVAL, f"resp: {resp}" + assert row.values[4].getType() == ttypes.Value.DTVAL, f"resp: {resp}" break assert session_id != 0 @@ -99,6 +99,8 @@ def test_sessions(self): resp = client_ok.execute('USE nba') self.check_resp_succeeded(resp) + # wait for session sync. + time.sleep(3) resp = self.execute('SHOW SESSION {}'.format(session_id)) self.check_resp_succeeded(resp) expect_col_names = ['VariableName', 'Value'] diff --git a/tests/tck/conftest.py b/tests/tck/conftest.py index 3f0ead037a8..1a0abf392b3 100644 --- a/tests/tck/conftest.py +++ b/tests/tck/conftest.py @@ -10,8 +10,10 @@ import io import csv import re +import threading from nebula2.common.ttypes import Value, ErrorCode +from nebula2.data.DataObject import ValueWrapper from pytest_bdd import given, parsers, then, when from tests.common.dataset_printer import DataSetPrinter @@ -34,6 +36,8 @@ rparse = functools.partial(parsers.re) example_pattern = re.compile(r"<(\w+)>") +register_dict = {} +register_lock = threading.Lock() def normalize_outline_scenario(request, name): for group in example_pattern.findall(name): @@ -76,7 +80,7 @@ def wait_all_jobs_finished(sess, jobs=[]): times = 4 * get_running_jobs(sess) while jobs and times > 0: jobs = [job for job in jobs if not is_job_finished(sess, job)] - time.sleep(0.5) + time.sleep(1) times -= 1 return len(jobs) == 0 @@ -283,8 +287,9 @@ def cmp_dataset( order: bool, strict: bool, contains=CmpType.EQUAL, + first_n_records=-1, hashed_columns=[], -) -> None: +): rs = graph_spaces['result_set'] ngql = graph_spaces['ngql'] check_resp(rs, ngql) @@ -298,6 +303,7 @@ def cmp_dataset( dscmp = DataSetComparator(strict=strict, order=order, contains=contains, + first_n_records=first_n_records, decode_type=rs._decode_type, vid_fn=vid_fn) @@ -333,6 +339,7 @@ def rowp(ds, i): f"vid_fn: {vid_fn}", ] assert res, "\n".join(msg) + return rds @then(parse("define some list variables:\n{text}")) @@ -478,3 +485,31 @@ def executing_query(query, index, graph_spaces, session_from_first_conn_pool, se exec_query(request, ngql, session_from_first_conn_pool, graph_spaces) else: exec_query(request, ngql, session_from_second_conn_pool, graph_spaces) + +@then(parse("the result should be, the first {n:d} records in order, and register {column_name} as a list named {key}:\n{result}")) +def result_should_be_in_order_and_register_key(n, column_name, key, request, result, graph_spaces): + assert n > 0, f"The records number should be an positive integer: {n}" + result_ds = cmp_dataset(request, graph_spaces, result, order=True, strict=True, contains=CmpType.CONTAINS, first_n_records=n) + register_result_key(request.node.name, result_ds, column_name, key) + +def register_result_key(test_name, result_ds, column_name, key): + if column_name.encode() not in result_ds.column_names: + assert False, f"{column_name} not in result columns {result_ds.column_names}." + col_index = result_ds.column_names.index(column_name.encode()) + val = [row.values[col_index] for row in result_ds.rows] + register_lock.acquire() + register_dict[test_name + key] = val; + register_lock.release() + +@when(parse("executing query, fill replace holders with element index of {indices} in {keys}:\n{query}")) +def executing_query_with_params(query, indices, keys, graph_spaces, session, request): + indices_list=[int(v) for v in indices.split(",")] + key_list=[request.node.name+key for key in keys.split(",")] + assert len(indices_list) == len(key_list), f"Length not match for keys and indices: {keys} <=> {indices}" + vals = [] + register_lock.acquire() + for (key, index) in zip (key_list, indices_list): + vals.append(ValueWrapper(register_dict[key][index])) + register_lock.release() + ngql = combine_query(query).format(*vals) + exec_query(request, ngql, session, graph_spaces) diff --git a/tests/tck/features/index/Index.IntVid.feature b/tests/tck/features/index/Index.IntVid.feature index a6aa4927ebf..a5534596fcd 100644 --- a/tests/tck/features/index/Index.IntVid.feature +++ b/tests/tck/features/index/Index.IntVid.feature @@ -69,21 +69,6 @@ Feature: IndexTest_Vid_Int hash("Tom"): ("Tom", 18, 11.11, `timestamp`("2000-10-10T10:00:00")) """ Then the execution should be successful - When executing query: - """ - REBUILD TAG INDEX single_tag_index; - """ - Then the execution should be successful - When executing query: - """ - REBUILD TAG INDEX single_tag_index OFFLINE; - """ - Then a SyntaxError should be raised at runtime: - When executing query: - """ - REBUILD TAG INDEX multi_tag_index; - """ - Then the execution should be successful When executing query: """ REBUILD TAG INDEX multi_tag_index OFFLINE; @@ -222,21 +207,6 @@ Feature: IndexTest_Vid_Int hash("May") -> hash("Tim"): ("Like", 18, 11.11, `timestamp`("2000-10-10T10:00:00")) """ Then the execution should be successful - When executing query: - """ - REBUILD EDGE INDEX single_edge_index - """ - Then the execution should be successful - When executing query: - """ - REBUILD EDGE INDEX single_edge_index OFFLINE; - """ - Then a SyntaxError should be raised at runtime: - When executing query: - """ - REBUILD EDGE INDEX multi_edge_1_index - """ - Then the execution should be successful When executing query: """ REBUILD EDGE INDEX multi_edge_1_index OFFLINE; @@ -618,7 +588,7 @@ Feature: IndexTest_Vid_Int """ Then the result should be, in any order: | Name | Index Status | - And wait 3 seconds + And wait 6 seconds When submit a job: """ REBUILD TAG INDEX tag_index_status @@ -666,7 +636,7 @@ Feature: IndexTest_Vid_Int """ Then the result should be, in any order: | Name | Index Status | - And wait 3 seconds + And wait 6 seconds When submit a job: """ REBUILD EDGE INDEX edge_index_status diff --git a/tests/tck/features/index/Index.feature b/tests/tck/features/index/Index.feature index 0813917eacf..432dd06153c 100644 --- a/tests/tck/features/index/Index.feature +++ b/tests/tck/features/index/Index.feature @@ -69,21 +69,6 @@ Feature: IndexTest_Vid_String "Tom": ("Tom", 18, 11.11, `timestamp`("2000-10-10T10:00:00")) """ Then the execution should be successful - When executing query: - """ - REBUILD TAG INDEX single_tag_index; - """ - Then the execution should be successful - When executing query: - """ - REBUILD TAG INDEX single_tag_index OFFLINE; - """ - Then a SyntaxError should be raised at runtime: - When executing query: - """ - REBUILD TAG INDEX multi_tag_index; - """ - Then the execution should be successful When executing query: """ REBUILD TAG INDEX multi_tag_index OFFLINE; @@ -222,21 +207,6 @@ Feature: IndexTest_Vid_String "May" -> "Tim": ("Like", 18, 11.11, `timestamp`("2000-10-10T10:00:00")) """ Then the execution should be successful - When executing query: - """ - REBUILD EDGE INDEX single_edge_index - """ - Then the execution should be successful - When executing query: - """ - REBUILD EDGE INDEX single_edge_index OFFLINE; - """ - Then a SyntaxError should be raised at runtime: - When executing query: - """ - REBUILD EDGE INDEX multi_edge_1_index - """ - Then the execution should be successful When executing query: """ REBUILD EDGE INDEX multi_edge_1_index OFFLINE; @@ -557,7 +527,7 @@ Feature: IndexTest_Vid_String CREATE TAG INDEX single_person_index ON tag_1(col1) """ Then the execution should be successful - And wait 3 seconds + And wait 6 seconds When try to execute query: """ INSERT VERTEX @@ -576,7 +546,7 @@ Feature: IndexTest_Vid_String CREATE TAG INDEX single_person_index2 ON tag_1(col5) """ Then the execution should be successful - And wait 3 seconds + And wait 6 seconds When try to execute query: """ INSERT VERTEX @@ -624,7 +594,7 @@ Feature: IndexTest_Vid_String """ Then the result should be, in any order: | Name | Index Status | - And wait 3 seconds + And wait 6 seconds When submit a job: """ REBUILD TAG INDEX tag_index_status @@ -672,7 +642,7 @@ Feature: IndexTest_Vid_String """ Then the result should be, in any order: | Name | Index Status | - And wait 3 seconds + And wait 6 seconds When submit a job: """ REBUILD EDGE INDEX edge_index_status diff --git a/tests/tck/features/index/TagEdgeIndex.feature b/tests/tck/features/index/TagEdgeIndex.feature index 266cb3c14b8..2412243eb56 100644 --- a/tests/tck/features/index/TagEdgeIndex.feature +++ b/tests/tck/features/index/TagEdgeIndex.feature @@ -80,20 +80,10 @@ Feature: tag and edge index tests from pytest CREATE TAG INDEX disorder_tag_index ON tag_1(col3, col2) """ Then the execution should be successful - And wait 3 seconds + And wait 6 seconds When submit a job: """ - REBUILD TAG INDEX single_tag_index - """ - Then wait the job to finish - When submit a job: - """ - REBUILD TAG INDEX multi_tag_index - """ - Then wait the job to finish - When submit a job: - """ - REBUILD TAG INDEX disorder_tag_index + REBUILD TAG INDEX single_tag_index, multi_tag_index, disorder_tag_index """ Then wait the job to finish When executing query: @@ -326,21 +316,11 @@ Feature: tag and edge index tests from pytest CREATE EDGE INDEX disorder_edge_index ON edge_1(col3, col2) """ Then the execution should be successful - And wait 3 seconds + And wait 6 seconds # Rebuild Edge Index When submit a job: """ - REBUILD EDGE INDEX single_edge_index - """ - Then wait the job to finish - When submit a job: - """ - REBUILD EDGE INDEX multi_edge_index - """ - Then wait the job to finish - When submit a job: - """ - REBUILD EDGE INDEX disorder_edge_index + REBUILD EDGE INDEX single_edge_index, multi_edge_index, disorder_edge_index """ Then wait the job to finish When executing query: diff --git a/tests/tck/features/insert/InsertEdgeOnDiffParts.feature b/tests/tck/features/insert/InsertEdgeOnDiffParts.feature new file mode 100644 index 00000000000..08c2d63ef79 --- /dev/null +++ b/tests/tck/features/insert/InsertEdgeOnDiffParts.feature @@ -0,0 +1,40 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. +Feature: Insert vertex and edge with if not exists + + Scenario: insert edge with default value + Given an empty graph + And create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(20) | + And having executed: + """ + CREATE TAG IF NOT EXISTS V(); + CREATE EDGE IF NOT EXISTS E(rank timestamp default timestamp()); + """ + When try to execute query: + """ + INSERT VERTEX V() VALUES "v1":() + """ + Then the execution should be successful + When try to execute query: + """ + INSERT VERTEX V() VALUES "v2":() + """ + Then the execution should be successful + When try to execute query: + """ + INSERT EDGE E() VALUES "v1"->"v2":() + """ + Then the execution should be successful + When executing query: + """ + (GO FROM "v1" over E yield E.rank union GO FROM "v2" over E REVERSELY yield E.rank) | yield count(*) AS count + """ + Then the result should be, in any order: + | count | + | 1 | + And drop the used space diff --git a/tests/tck/features/job/SpaceRequire.feature b/tests/tck/features/job/SpaceRequire.feature deleted file mode 100644 index dbe07b650f9..00000000000 --- a/tests/tck/features/job/SpaceRequire.feature +++ /dev/null @@ -1,29 +0,0 @@ -Feature: Submit job space requirements - - Scenario: submit job require space - Given an empty graph - When executing query: - """ - SUBMIT JOB COMPACT; - """ - Then a SemanticError should be raised at runtime: - When executing query: - """ - SUBMIT JOB FLUSH; - """ - Then a SemanticError should be raised at runtime: - When executing query: - """ - SUBMIT JOB STATS; - """ - Then a SemanticError should be raised at runtime: - When executing query: - """ - REBUILD TAG INDEX not_exists_index; - """ - Then a SemanticError should be raised at runtime: - When executing query: - """ - REBUILD EDGE INDEX not_exists_index; - """ - Then a SemanticError should be raised at runtime: diff --git a/tests/tck/features/lookup/EdgeIndexFullScan.feature b/tests/tck/features/lookup/EdgeIndexFullScan.feature index 22dfc781a31..293eaa339e3 100644 --- a/tests/tck/features/lookup/EdgeIndexFullScan.feature +++ b/tests/tck/features/lookup/EdgeIndexFullScan.feature @@ -12,7 +12,6 @@ Feature: Lookup edge index full scan """ CREATE EDGE edge_1(col1_str string, col2_int int); """ - And wait 3 seconds # index on col1_str And having executed: """ @@ -23,7 +22,7 @@ Feature: Lookup edge index full scan """ CREATE EDGE INDEX col2_int_index ON edge_1(col2_int) """ - And wait 3 seconds + And wait 6 seconds And having executed: """ INSERT EDGE @@ -33,7 +32,6 @@ Feature: Lookup edge index full scan '102'->'103':('Yellow', 22), '103'->'101':('Blue', 33); """ - And wait 3 seconds Scenario: Edge with relational RegExp filter[1] When executing query: diff --git a/tests/tck/features/match/SeekById.intVid.feature b/tests/tck/features/match/SeekById.intVid.feature index 94af09015fb..d1d427c3229 100644 --- a/tests/tck/features/match/SeekById.intVid.feature +++ b/tests/tck/features/match/SeekById.intVid.feature @@ -279,22 +279,17 @@ Feature: Match seek by id """ CREATE TAG player(name string, age int); """ - When try to execute query: - """ - INSERT VERTEX player(name, age) VALUES -100:("Tim", 32); - """ - Then the execution should be successful When executing query: """ CREATE TAG INDEX player_name_index ON player(name(10)); """ Then the execution should be successful And wait 6 seconds - When submit a job: + When try to execute query: """ - REBUILD TAG INDEX player_name_index; + INSERT VERTEX player(name, age) VALUES -100:("Tim", 32); """ - Then wait the job to finish + Then the execution should be successful When executing query: """ MATCH (v) WHERE id(v) == -100 RETURN v; diff --git a/tests/tck/features/schema/Schema.feature b/tests/tck/features/schema/Schema.feature index 46cb9e011de..4678480aec8 100644 --- a/tests/tck/features/schema/Schema.feature +++ b/tests/tck/features/schema/Schema.feature @@ -577,7 +577,7 @@ Feature: Insert string vid of vertex and edge """ INSERT EDGE e() VALUES "1"->"2":() """ - Then a ExecutionError should be raised at runtime: Storage Error: The not null field doesn't have a default value. + Then a SemanticError should be raised at runtime: The property `description' is not nullable and has no default value. # test alter edge with timestamp default When executing query: """ diff --git a/tests/tck/job/Job.feature b/tests/tck/job/Job.feature new file mode 100644 index 00000000000..fb129036420 --- /dev/null +++ b/tests/tck/job/Job.feature @@ -0,0 +1,106 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. +Feature: Submit job space requirements + + Scenario: submit job require space + Given an empty graph + When executing query: + """ + SUBMIT JOB COMPACT; + """ + Then a SemanticError should be raised at runtime: + When executing query: + """ + SUBMIT JOB FLUSH; + """ + Then a SemanticError should be raised at runtime: + When executing query: + """ + SUBMIT JOB STATS; + """ + Then a SemanticError should be raised at runtime: + When executing query: + """ + REBUILD TAG INDEX not_exists_index; + """ + Then a SemanticError should be raised at runtime: + When executing query: + """ + REBUILD EDGE INDEX not_exists_index; + """ + Then a SemanticError should be raised at runtime: + + Scenario: Not existed job + Given an empty graph + When executing query: + """ + SHOW JOB 123456; + """ + Then a ExecutionError should be raised at runtime: Key not existed! + When executing query: + """ + STOP JOB 123456; + """ + Then a ExecutionError should be raised at runtime: Key not existed! + + Scenario: Submit and show jobs + Given create a space with following options: + | partition_num | 9 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(20) | + And wait 6 seconds + When executing query: + """ + SUBMIT JOB COMPACT; + """ + Then the result should be, in any order: + | New Job Id | + | /\d+/ | + And wait 1 seconds + When executing query: + """ + SUBMIT JOB FLUSH; + """ + Then the result should be, in any order: + | New Job Id | + | /\d+/ | + And wait 1 seconds + When executing query: + """ + SUBMIT JOB STATS; + """ + Then the result should be, in any order: + | New Job Id | + | /\d+/ | + And wait 10 seconds + When executing query: + """ + SHOW JOBS; + """ + Then the result should be, the first 3 records in order, and register Job Id as a list named job_id: + | Job Id | Command | Status | Start Time | Stop Time | + | /\d+/ | "STATS" | "FINISHED" | /\w+/ | /\w+/ | + | /\d+/ | "FLUSH" | "FINISHED" | /\w+/ | /\w+/ | + | /\d+/ | "COMPACT" | "FINISHED" | /\w+/ | /\w+/ | + When executing query, fill replace holders with element index of 0 in job_id: + """ + SHOW JOB {}; + """ + Then the result should be, in order: + | Job Id(TaskId) | Command(Dest) | Status | Start Time | Stop Time | + | /\d+/ | "STATS" | "FINISHED" | /\w+/ | /\w+/ | + | /\d+/ | /\w+/ | "FINISHED" | /\w+/ | /\w+/ | + When executing query, fill replace holders with element index of 0 in job_id: + """ + STOP JOB {}; + """ + Then an ExecutionError should be raised at runtime: Save job failure! + +# This is skipped becuase it is hard to simulate the situation +# When executing query: +# """ +# RECOVER JOB; +# """ +# Then the result should be successful diff --git a/tests/tck/steps/test_jobs.py b/tests/tck/steps/test_jobs.py new file mode 100644 index 00000000000..26a53060d95 --- /dev/null +++ b/tests/tck/steps/test_jobs.py @@ -0,0 +1,9 @@ +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +from pytest_bdd import scenarios + + +scenarios('job/Job.feature')