Rollback vertex key (#4629)

* remove upgrade tool * rollback * fetch/go/insert tck * match tck * Update GoYieldVertexEdge.feature * Update GoYieldVertexEdge.feature * Update GO.feature * change use_vertex_key to graph_use_vertex_key * Update GraphFlags.h * Update GraphFlags.cpp Co-authored-by: hs.zhang <22708345+cangfengzhs@users.noreply.github.com> Co-authored-by: jimingquan <mingquan.ji@vesoft.com>
vesoft-inc · Sep 21, 2022 · 545137a · 545137a
1 parent 7d5e93f
commit 545137a
Show file tree

Hide file tree

Showing 34 changed files with 81 additions and 2,895 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -5,4 +5,4 @@ Following docker images will be ready in production.
 - [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
 - [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
 - [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
-- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader
+- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump and meta_dump
diff --git a/src/common/utils/NebulaKeyUtils.cpp b/src/common/utils/NebulaKeyUtils.cpp
@@ -253,7 +253,6 @@ std::vector<std::string> NebulaKeyUtils::snapshotPrefix(PartitionID partId) {
   if (partId == 0) {
     result.emplace_back("");
   } else {
-    result.emplace_back(vertexPrefix(partId));
     result.emplace_back(tagPrefix(partId));
     result.emplace_back(edgePrefix(partId));
     result.emplace_back(IndexKeyUtils::indexPrefix(partId));

diff --git a/src/common/utils/NebulaKeyUtils.h b/src/common/utils/NebulaKeyUtils.h
@@ -155,6 +155,12 @@ class NebulaKeyUtils final {
     return static_cast<NebulaKeyType>(type) == NebulaKeyType::kEdge;
   }
 
+  static bool isVertex(const folly::StringPiece& rawKey) {
+    constexpr int32_t len = static_cast<int32_t>(sizeof(NebulaKeyType));
+    auto type = readInt<uint32_t>(rawKey.data(), len) & kTypeMask;
+    return static_cast<NebulaKeyType>(type) == NebulaKeyType::kVertex;
+  }
+
   static bool isLock(size_t vIdLen, const folly::StringPiece& rawKey) {
     return isEdge(vIdLen, rawKey, kLockVersion);
   }

diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp
@@ -103,3 +103,5 @@ DEFINE_uint32(
     gc_worker_size,
     0,
     "Background garbage clean workers, default number is 0 which means using hardware core size.");
+
+DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key");
diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h
@@ -65,4 +65,7 @@ DECLARE_int32(max_job_size);
 
 DECLARE_bool(enable_async_gc);
 DECLARE_uint32(gc_worker_size);
+
+DECLARE_bool(graph_use_vertex_key);
+
 #endif  // GRAPH_GRAPHFLAGS_H_
diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp
@@ -46,6 +46,9 @@ Status InsertVerticesValidator::check() {
   }
 
   auto tagItems = sentence->tagItems();
+  if (!FLAGS_graph_use_vertex_key && tagItems.empty()) {
+    return Status::SemanticError("Insert vertex is forbidden, please speicify the tag");
+  }
 
   schemas_.reserve(tagItems.size());
 

diff --git a/src/storage/CompactionFilter.h b/src/storage/CompactionFilter.h
@@ -14,6 +14,7 @@
 #include "common/utils/OperationKeyUtils.h"
 #include "kvstore/CompactionFilter.h"
 #include "storage/CommonUtils.h"
+#include "storage/StorageFlags.h"
 
 namespace nebula {
 namespace storage {
@@ -36,6 +37,8 @@ class StorageCompactionFilter final : public kvstore::KVFilter {
       return !edgeValid(spaceId, key, val);
     } else if (IndexKeyUtils::isIndexKey(key)) {
       return !indexValid(spaceId, key, val);
+    } else if (!FLAGS_use_vertex_key && NebulaKeyUtils::isVertex(key)) {
+      return true;
     } else if (NebulaKeyUtils::isLock(vIdLen_, key)) {
       return !lockValid(spaceId, key);
     } else {

diff --git a/src/storage/StorageFlags.cpp b/src/storage/StorageFlags.cpp
@@ -46,3 +46,5 @@ DEFINE_bool(query_concurrently,
             false,
             "whether to run query of each part concurrently, only lookup and "
             "go are supported");
+
+DEFINE_bool(use_vertex_key, false, "whether allow insert or query the vertex key");
diff --git a/src/storage/StorageFlags.h b/src/storage/StorageFlags.h
@@ -34,4 +34,6 @@ DECLARE_int32(max_edge_returned_per_vertex);
 
 DECLARE_bool(query_concurrently);
 
+DECLARE_bool(use_vertex_key);
+
 #endif  // STORAGE_STORAGEFLAGS_H_
diff --git a/src/storage/admin/StatsTask.cpp b/src/storage/admin/StatsTask.cpp
@@ -111,8 +111,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
   std::unique_ptr<kvstore::KVIterator> tagIter;
   auto edgePrefix = NebulaKeyUtils::edgePrefix(part);
   std::unique_ptr<kvstore::KVIterator> edgeIter;
-  auto vertexPrefix = NebulaKeyUtils::vertexPrefix(part);
-  std::unique_ptr<kvstore::KVIterator> vertexIter;
 
   // When the storage occurs leader change, continue to read data from the
   // follower instead of reporting an error.
@@ -126,11 +124,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     LOG(INFO) << "Stats task failed";
     return ret;
   }
-  ret = env_->kvstore_->prefix(spaceId, part, vertexPrefix, &vertexIter, true);
-  if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
-    LOG(INFO) << "Stats task failed";
-    return ret;
-  }
   std::unordered_map<TagID, int64_t> tagsVertices;
   std::unordered_map<EdgeType, int64_t> edgetypeEdges;
   std::unordered_map<PartitionID, int64_t> positiveRelevancy;
@@ -173,6 +166,7 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
       tagIter->next();
       continue;
     }
+    spaceVertices++;
     tagsVertices[tagId] += 1;
     tagIter->next();
     sleepIfScannedSomeRecord(++countToSleep);
@@ -227,11 +221,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
     edgeIter->next();
     sleepIfScannedSomeRecord(++countToSleep);
   }
-  while (vertexIter && vertexIter->valid()) {
-    spaceVertices++;
-    vertexIter->next();
-    sleepIfScannedSomeRecord(++countToSleep);
-  }
   nebula::meta::cpp2::StatsItem statsItem;
 
   // convert tagId/edgeType to tagName/edgeName

diff --git a/src/storage/exec/GetPropNode.h b/src/storage/exec/GetPropNode.h
@@ -42,18 +42,23 @@ class GetTagPropNode : public QueryNode<VertexID> {
       return ret;
     }
 
-    // if none of the tag node and vertex valid, do not emplace the row
+    // If none of the tag node valid, will check vertex key if use_vertex_key is true,
+    // do not emplace the row if the flag is false
     if (!std::any_of(tagNodes_.begin(), tagNodes_.end(), [](const auto& tagNode) {
           return tagNode->valid();
         })) {
-      auto kvstore = context_->env()->kvstore_;
-      auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId);
-      std::string value;
-      ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value);
-      if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
+      if (FLAGS_use_vertex_key) {
+        auto kvstore = context_->env()->kvstore_;
+        auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId);
+        std::string value;
+        ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value);
+        if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
+          return nebula::cpp2::ErrorCode::SUCCEEDED;
+        } else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
+          return ret;
+        }
+      } else {
         return nebula::cpp2::ErrorCode::SUCCEEDED;
-      } else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
-        return ret;
       }
     }
 

diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h
@@ -488,7 +488,6 @@ class UpdateTagNode : public UpdateNode<VertexID> {
       }
     }
     // step 3, insert new vertex data
-    batchHolder->put(NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId), "");
     batchHolder->put(std::move(key_), std::move(nVal));
     return encodeBatchValue(batchHolder->getBatch());
   }

diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp
@@ -81,7 +81,6 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
         code = nebula::cpp2::ErrorCode::E_INVALID_VID;
         break;
       }
-      data.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid), "");
       for (auto& newTag : newTags) {
         auto tagId = newTag.get_tag_id();
         VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId;
@@ -160,7 +159,6 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re
         break;
       }
 
-      verticeData.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid));
       for (const auto& newTag : newTags) {
         auto tagId = newTag.get_tag_id();
         VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId;

diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
@@ -65,4 +65,3 @@ nebula_add_subdirectory(simple-kv-verify)
 endif()
 nebula_add_subdirectory(meta-dump)
 nebula_add_subdirectory(db-dump)
-nebula_add_subdirectory(db-upgrade)
diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt