From c50c34a9042f115d1a8d5e53b27fc52aa6df5f41 Mon Sep 17 00:00:00 2001 From: Linary Date: Wed, 27 Mar 2019 16:22:34 +0800 Subject: [PATCH] Support pagination when single index query (#328) Support pagination when single index query Change-Id: Iab355a9f68cebd8ebc666cf126842eb19fa40240 --- hugegraph-api/pom.xml | 4 +- .../baidu/hugegraph/api/graph/EdgeAPI.java | 13 +- .../baidu/hugegraph/api/graph/VertexAPI.java | 13 +- .../baidu/hugegraph/api/job/GremlinAPI.java | 3 +- .../baidu/hugegraph/version/ApiVersion.java | 3 +- hugegraph-cassandra/pom.xml | 2 +- .../store/cassandra/CassandraTable.java | 47 +- hugegraph-core/pom.xml | 4 +- .../hugegraph/backend/page/IdHolder.java | 103 ++++ .../hugegraph/backend/page/IdHolderList.java | 71 +++ .../backend/page/PageEntryIterator.java | 107 ++++ .../baidu/hugegraph/backend/page/PageIds.java | 50 ++ .../hugegraph/backend/page/PageState.java | 97 +++ .../hugegraph/backend/page/QueryList.java | 262 ++++++++ .../backend/query/ConditionQuery.java | 1 - .../backend/query/ConditionQueryFlatten.java | 14 +- .../backend/query/IdPrefixQuery.java | 4 + .../baidu/hugegraph/backend/query/Query.java | 3 +- .../serializer/BinaryInlineSerializer.java | 2 +- .../backend/serializer/BinarySerializer.java | 70 ++- .../backend/store/BackendStoreSystemInfo.java | 5 +- .../backend/tx/AbstractTransaction.java | 1 + .../backend/tx/GraphIndexTransaction.java | 189 +++--- .../backend/tx/GraphTransaction.java | 42 +- .../baidu/hugegraph/config/CoreOptions.java | 8 + .../baidu/hugegraph/version/CoreVersion.java | 2 +- hugegraph-dist/pom.xml | 2 +- hugegraph-example/pom.xml | 2 +- hugegraph-hbase/pom.xml | 2 +- .../backend/store/hbase/HbaseSerializer.java | 2 +- .../backend/store/hbase/HbaseSessions.java | 11 + .../store/hbase/HbaseStoreProvider.java | 3 +- .../backend/store/hbase/HbaseTable.java | 12 +- .../backend/store/hbase/HbaseTables.java | 39 +- hugegraph-mysql/pom.xml | 2 +- .../backend/store/mysql/MysqlTable.java | 3 + .../backend/store/mysql/MysqlTables.java | 21 - hugegraph-palo/pom.xml | 2 +- hugegraph-rocksdb/pom.xml | 2 +- .../backend/store/rocksdb/RocksDBTable.java | 3 +- hugegraph-scylladb/pom.xml | 2 +- hugegraph-test/pom.xml | 2 +- .../baidu/hugegraph/core/VertexCoreTest.java | 570 +++++++++++++++++- .../hugegraph/core/VertexLabelCoreTest.java | 4 +- .../src/main/resources/hugegraph.properties | 2 + pom.xml | 2 +- 46 files changed, 1584 insertions(+), 224 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolder.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageIds.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageState.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java diff --git a/hugegraph-api/pom.xml b/hugegraph-api/pom.xml index 74ac2d4195..7082c4ad44 100644 --- a/hugegraph-api/pom.xml +++ b/hugegraph-api/pom.xml @@ -5,7 +5,7 @@ hugegraph com.baidu.hugegraph - 0.9.1 + 0.9.2 4.0.0 @@ -86,7 +86,7 @@ - 0.34.0.0 + 0.35.0.0 diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java index 75ed3a4761..ba54eba04d 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/EdgeAPI.java @@ -222,17 +222,20 @@ public String list(@Context GraphManager manager, LOG.debug("Graph [{}] query edges by vertex: {}, direction: {}, " + "label: {}, properties: {}, offset: {}, page: {}, limit: {}", vertexId, direction, label, properties, offset, page, limit); + + Map props = parseProperties(properties); if (page != null) { E.checkArgument(vertexId == null && direction == null && - label == null && properties == null && offset == 0, - "Not support quering edges based on paging and " + - "[vertex, direction, label, properties, offset] " + - "together"); + offset == 0, + "Not support querying edges based on paging " + + "and [vertex, direction, offset] together"); + E.checkArgument(props.size() <= 1, + "Not support querying edges based on paging " + + "and more than one property"); } Id vertex = VertexAPI.checkAndParseVertexId(vertexId); Direction dir = parseDirection(direction); - Map props = parseProperties(properties); HugeGraph g = graph(manager, graph); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java index 8479f46712..ee0823c01e 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java @@ -173,13 +173,16 @@ public String list(@Context GraphManager manager, LOG.debug("Graph [{}] query vertices by label: {}, properties: {}, " + "offset: {}, page: {}, limit: {}", graph, label, properties, offset, page, limit); - if (page != null) { - E.checkArgument(label == null && properties == null && offset == 0, - "Not support quering vertices based on paging " + - "and [label, properties, offset] together"); - } Map props = parseProperties(properties); + if (page != null) { + E.checkArgument(offset == 0, + "Not support querying vertices based on paging " + + "and offset together"); + E.checkArgument(props.size() <= 1, + "Not support querying vertices based on paging " + + "and more than one property"); + } HugeGraph g = graph(manager, graph); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java index 5f76c7f7e3..46c3988cd5 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java @@ -59,8 +59,7 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; - -import jersey.repackaged.com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap; @Path("graphs/{graph}/jobs/gremlin") @Singleton diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java index 9ef7222e22..9a627212a3 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java @@ -78,10 +78,11 @@ public final class ApiVersion { * [0.32] Issue-250: Keep depth and degree consistent for traverser api * [0.33] Issue-305: Implement customized paths and crosspoints RESTful API * [0.34] Issue-307: Let VertexAPI use simplified property serializer + * [0.35] Issue-287: Support pagination when do index query */ // The second parameter of Version.of() is for IDE running without JAR - public static final Version VERSION = Version.of(ApiVersion.class, "0.34"); + public static final Version VERSION = Version.of(ApiVersion.class, "0.35"); public static final void check() { // Check version of hugegraph-core. Firstly do check from version 0.3 diff --git a/hugegraph-cassandra/pom.xml b/hugegraph-cassandra/pom.xml index 5f98a01252..fa8352fe2b 100644 --- a/hugegraph-cassandra/pom.xml +++ b/hugegraph-cassandra/pom.xml @@ -5,7 +5,7 @@ hugegraph com.baidu.hugegraph - 0.9.1 + 0.9.2 4.0.0 diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java index ae8c80b974..41b45df32e 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java @@ -127,25 +127,6 @@ protected List query2Select(String table, Query query) { if (query.conditions().isEmpty()) { // Query only by id + this.setPageState(query, ids); LOG.debug("Query only by id(s): {}", ids); return ids; } else { - List(ids.size()); + List selects) { + if (query.limit() == Query.NO_LIMIT) { + return; + } + for (Select select : selects) { + long total = query.total(); + String page = query.page(); + if (page == null) { + // Set limit + select.limit((int) total); + } else { + select.setFetchSize((int) total); + // It's the first time if page is empty + if (!page.isEmpty()) { + try { + select.setPagingState(PagingState.fromString(page)); + } catch (PagingStateException e) { + throw new BackendException(e); + } + } + } + } + } + protected List