Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support pagination when single index query #328

Merged
merged 8 commits into from
Mar 27, 2019
Merged

Support pagination when single index query #328

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d741e5c
Support pagination when single index query
Linary Jan 16, 2019
b2223c7
Make some modifications
Linary Mar 13, 2019
4a25e2c
Refactor IdHolder, QueryHolder and other page related classes
Linary Mar 14, 2019
7a1ab1c
Modification on GraphTransaction and GraphIndexTransaction
Linary Mar 15, 2019
274d28e
Modification on hbase table
Linary Mar 15, 2019
15d7395
rename index.page_size to query.page_size
Linary Mar 18, 2019
a565f26
Modified on IdHolder and related classes
Linary Mar 22, 2019
7d49b57
Combine two methods into doIndexQuery
Linary Mar 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,10 @@ public PageIds fetchNext(String page, long pageSize) {
this.query.limit(pageSize);

PageIds result = this.idsFetcher.get();
Linary marked this conversation as resolved.
Show resolved Hide resolved
if (result == null) {
return null;
}
this.ids = result.ids();

assert this.ids != null;
if (this.ids.size() != this.query.limit() || result.page() == null) {
this.exhausted = true;
}
Expand Down
21 changes: 18 additions & 3 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@

import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;

import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.IdQuery;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.config.CoreOptions;
import com.baidu.hugegraph.iterator.FlatMapperIterator;
import com.baidu.hugegraph.iterator.Metadatable;
import com.baidu.hugegraph.util.CollectionUtil;
Expand All @@ -40,13 +42,15 @@

public final class QueryList {

private final HugeGraph graph;
private final Query parent;
// The size of each page fetched by the inner page
private final Function<Query, Iterator<BackendEntry>> fetcher;
private final List<QueryHolder> queries;

public QueryList(Query parent,
public QueryList(HugeGraph graph, Query parent,
Function<Query, Iterator<BackendEntry>> fetcher) {
this.graph = graph;
this.parent = parent;
this.fetcher = fetcher;
this.queries = new ArrayList<>();
Expand Down Expand Up @@ -85,7 +89,18 @@ public boolean empty() {
return this.queries.isEmpty();
}

public Iterator<BackendEntry> fetchAll() {
public Iterator<BackendEntry> fetch() {
assert !queries.isEmpty();
if (this.parent.paging()) {
int pageSize = this.graph.configuration()
.get(CoreOptions.INDEX_PAGE_SIZE);
return new PageEntryIterator(this, pageSize);
} else {
return this.fetchAll();
}
}

private Iterator<BackendEntry> fetchAll() {
Linary marked this conversation as resolved.
Show resolved Hide resolved
return new FlatMapperIterator<>(this.queries.iterator(), q -> {
return q.iterator();
});
Expand Down Expand Up @@ -210,7 +225,7 @@ public Iterator<BackendEntry> iterator() {
public PageIterator iterator(int index, String page, long pageSize) {
IdHolder holder = this.holders.get(index);
PageIds pageIds = holder.fetchNext(page, pageSize);
if (pageIds == null || pageIds.ids().isEmpty()) {
if (pageIds == null) {
Linary marked this conversation as resolved.
Show resolved Hide resolved
return PageIterator.EMPTY;
}
IdQuery query = new IdQuery(parent.resultType(), pageIds.ids());
Expand Down
2 changes: 1 addition & 1 deletion hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ private IdHolder doIndexQueryInPage(IndexLabel indexLabel,
}
Linary marked this conversation as resolved.
Show resolved Hide resolved
// If there is no data, the entries is not a Metadatable object
if (ids.isEmpty()) {
return new PageIds(ids, null);
return null;
}

E.checkState(entries instanceof Metadatable,
Expand Down
12 changes: 3 additions & 9 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.SplicingIdGenerator;
import com.baidu.hugegraph.backend.page.IdHolder;
import com.baidu.hugegraph.backend.page.PageEntryIterator;
import com.baidu.hugegraph.backend.page.QueryList;
import com.baidu.hugegraph.backend.query.Condition;
import com.baidu.hugegraph.backend.query.ConditionQuery;
Expand Down Expand Up @@ -314,7 +313,8 @@ public Iterator<BackendEntry> query(Query query) {
return super.query(query);
}

QueryList queries = new QueryList(query, q -> super.query(q));
QueryList queries = new QueryList(this.graph(), query,
q -> super.query(q));
for (ConditionQuery cq: ConditionQueryFlatten.flatten(
(ConditionQuery) query)) {
Query q = this.optimizeQuery(cq);
Expand All @@ -330,13 +330,7 @@ public Iterator<BackendEntry> query(Query query) {
}
}

if (query.paging() && !queries.empty()) {
int pageSize = this.config().get(CoreOptions.INDEX_PAGE_SIZE);
return new PageEntryIterator(queries, pageSize);
} else if (!queries.empty()) {
return queries.fetchAll();
}
return Collections.emptyIterator();
return !queries.empty() ? queries.fetch() : Collections.emptyIterator();
}

@Watched(prefix = "graph")
Expand Down