From 6a800ecd2153862f59f39e3052d82960b79ea61c Mon Sep 17 00:00:00 2001 From: liningrui Date: Wed, 9 Jan 2019 18:06:49 +0800 Subject: [PATCH] Support pagination when single index query Change-Id: I15c10c702078ff4a998378633e5cbb68cf2ac5fa --- .../baidu/hugegraph/api/job/GremlinAPI.java | 3 +- .../backend/query/ConditionQuery.java | 1 - .../backend/query/IdPrefixQuery.java | 4 + .../baidu/hugegraph/backend/query/Query.java | 6 +- .../serializer/BinaryEntryIterator.java | 60 ----- .../backend/serializer/BinarySerializer.java | 53 +++-- .../backend/serializer/PageState.java | 83 +++++++ .../backend/store/BackendEntryIterator.java | 7 +- .../baidu/hugegraph/backend/tx/EntireIds.java | 46 ++++ .../backend/tx/EntireIndexIdsHolder.java | 69 ++++++ .../backend/tx/GraphIndexTransaction.java | 202 +++++++++++----- .../backend/tx/GraphTransaction.java | 61 +++-- .../com/baidu/hugegraph/backend/tx/Ids.java | 29 +++ .../hugegraph/backend/tx/IndexIdsHolder.java | 52 +++++ .../backend/tx/IndexIdsHolderChain.java | 102 ++++++++ .../baidu/hugegraph/backend/tx/PagedIds.java | 53 +++++ .../backend/tx/PagedIndexIdsHolder.java | 113 +++++++++ .../baidu/hugegraph/config/CoreOptions.java | 8 + .../com/baidu/hugegraph/example/Example2.java | 221 +++++++----------- .../src/main/resources/hugegraph.properties | 7 +- .../backend/store/hbase/HbaseTable.java | 2 +- .../backend/store/rocksdb/RocksDBTable.java | 2 +- .../baidu/hugegraph/core/VertexCoreTest.java | 2 +- .../hugegraph/core/VertexLabelCoreTest.java | 4 +- .../src/main/resources/hugegraph.properties | 4 +- 25 files changed, 895 insertions(+), 299 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/PageState.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIds.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIndexIdsHolder.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/Ids.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolder.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolderChain.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIds.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIndexIdsHolder.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java index 13db6b7840..8df4c925c5 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/GremlinAPI.java @@ -59,8 +59,7 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.annotation.JsonProperty; - -import jersey.repackaged.com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap; @Path("graphs/{graph}/jobs/gremlin") @Singleton diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java index a83a755659..0551a6f276 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java @@ -408,7 +408,6 @@ public int optimized() { return this.optimizedType; } - public void registerResultsFilter(Function filter) { this.resultsFilter = filter; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java index 48cf77a9f9..70e83b7915 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdPrefixQuery.java @@ -39,6 +39,10 @@ public IdPrefixQuery(Query originQuery, Id prefix) { this(originQuery.resultType(), originQuery, prefix, true, prefix); } + public IdPrefixQuery(Query originQuery, Id start, Id prefix) { + this(originQuery.resultType(), originQuery, start, true, prefix); + } + public IdPrefixQuery(Query originQuery, Id start, boolean inclusive, Id prefix) { this(originQuery.resultType(), originQuery, start, inclusive, prefix); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java index 1ae2715438..94dfa1e4cd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java @@ -39,7 +39,8 @@ public class Query implements Cloneable { public static final long NO_LIMIT = Long.MAX_VALUE; public static final long NO_CAPACITY = -1L; - public static final long DEFAULT_CAPACITY = 800000L; // HugeGraph-777 + // TODO: Still using 80w or changing to a smaller value + public static final long DEFAULT_CAPACITY = 1000L; // HugeGraph-777 private HugeType resultType; private Map orders; @@ -265,8 +266,9 @@ public int hashCode() { @Override public String toString() { - return String.format("Query for %s offset=%d, limit=%d, order by %s", + return String.format("Query for %s page=%s, offset=%d, limit=%d, order by %s", this.resultType, + this.page, this.offset, this.limit, this.orders.toString()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryEntryIterator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryEntryIterator.java index 94aee65c9d..9cfc7698f7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryEntryIterator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryEntryIterator.java @@ -19,15 +19,12 @@ package com.baidu.hugegraph.backend.serializer; -import java.util.Base64; import java.util.function.BiFunction; -import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry.BackendIterator; import com.baidu.hugegraph.backend.store.BackendEntryIterator; -import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; public class BinaryEntryIterator extends BackendEntryIterator { @@ -140,61 +137,4 @@ private void skipPageOffset(String page) { this.skip(this.current, pagestate.offset()); } } - - public static class PageState { - - private final byte[] position; - private final int offset; - - public PageState(byte[] position, int offset) { - E.checkNotNull(position, "position"); - this.position = position; - this.offset = offset; - } - - public byte[] position() { - return this.position; - } - - public int offset() { - return this.offset; - } - - @Override - public String toString() { - return Base64.getEncoder().encodeToString(this.toBytes()); - } - - public byte[] toBytes() { - int length = 2 + this.position.length + BytesBuffer.INT_LEN; - BytesBuffer buffer = BytesBuffer.allocate(length); - buffer.writeBytes(this.position); - buffer.writeInt(this.offset); - return buffer.bytes(); - } - - public static PageState fromString(String page) { - byte[] bytes; - try { - bytes = Base64.getDecoder().decode(page); - } catch (Exception e) { - throw new BackendException("Invalid page: '%s'", e, page); - } - return fromBytes(bytes); - } - - public static PageState fromBytes(byte[] bytes) { - if (bytes.length == 0) { - // The first page - return new PageState(new byte[0], 0); - } - try { - BytesBuffer buffer = BytesBuffer.wrap(bytes); - return new PageState(buffer.readBytes(), buffer.readInt()); - } catch (Exception e) { - throw new BackendException("Invalid page: '0x%s'", - e, Bytes.toHex(bytes)); - } - } - } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index cbfc54c036..3ef3eba688 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -34,7 +34,6 @@ import com.baidu.hugegraph.backend.query.Condition.Relation; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.IdPrefixQuery; -import com.baidu.hugegraph.backend.query.IdQuery; import com.baidu.hugegraph.backend.query.IdRangeQuery; import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry.BinaryId; @@ -104,7 +103,6 @@ private BinaryBackendEntry newBackendEntry(HugeEdge edge) { return new BinaryBackendEntry(edge.type(), id); } - @SuppressWarnings("unused") private BinaryBackendEntry newBackendEntry(SchemaElement elem) { return newBackendEntry(elem.type(), elem.id()); } @@ -633,11 +631,26 @@ private Query writeStringIndexQuery(ConditionQuery query) { E.checkArgument(index != null, "Please specify the index label"); E.checkArgument(key != null, "Please specify the index key"); - Id id = formatIndexId(query.resultType(), index, key); - IdQuery idQuery = new IdQuery(query, id); - idQuery.limit(query.limit()); - idQuery.offset(query.offset()); - return idQuery; + Id prefix = formatIndexId(query.resultType(), index, key); + + Query newQuery; + /* + * If used paging and the page number is not empty, deserialize + * the page to id and use it as the starting row for this query + */ + if (query.paging() && !query.page().isEmpty()) { + byte[] position = PageState.fromString(query.page()).position(); + BinaryId start = new BinaryId(position, null); + newQuery = new IdPrefixQuery(query, start, prefix); + } else { + newQuery = new IdPrefixQuery(query, prefix); + } + if (query.paging()) { + newQuery.page(query.page()); + } + newQuery.limit(query.limit()); + newQuery.offset(query.offset()); + return newQuery; } private Query writeRangeIndexQuery(ConditionQuery query) { @@ -680,10 +693,11 @@ private Query writeRangeIndexQuery(ConditionQuery query) { HugeType type = query.resultType(); if (keyEq != null) { Id id = formatIndexId(type, index, keyEq); - IdQuery idQuery = new IdQuery(query, id); - idQuery.limit(query.limit()); - idQuery.offset(query.offset()); - return idQuery; + Query newQuery = new IdPrefixQuery(query, id); + newQuery.page(query.page()); + newQuery.limit(query.limit()); + newQuery.offset(query.offset()); + return newQuery; } if (keyMin == null) { @@ -704,15 +718,28 @@ private Query writeRangeIndexQuery(ConditionQuery query) { keyMinEq = true; } + Id start = min; + if (query.paging() && !query.page().isEmpty()) { + byte[] position = PageState.fromString(query.page()).position(); + start = new BinaryId(position, null); + } + + Query newQuery; if (keyMax == null) { Id prefix = formatIndexId(type, index, null); // Reset the first byte to make same length-prefix prefix.asBytes()[0] = min.asBytes()[0]; - return new IdPrefixQuery(query, min, keyMinEq, prefix); + newQuery = new IdPrefixQuery(query, start, keyMinEq, prefix); } else { Id max = formatIndexId(type, index, keyMax); - return new IdRangeQuery(query, min, keyMinEq, max, keyMaxEq); + newQuery = new IdRangeQuery(query, start, keyMinEq, max, keyMaxEq); + } + if (query.paging()) { + newQuery.page(query.page()); } + newQuery.limit(query.limit()); + newQuery.offset(query.offset()); + return newQuery; } private BinaryBackendEntry formatILDeletion(HugeIndex index) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/PageState.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/PageState.java new file mode 100644 index 0000000000..0a4d39bf9e --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/PageState.java @@ -0,0 +1,83 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.serializer; + +import java.util.Base64; + +import com.baidu.hugegraph.backend.BackendException; +import com.baidu.hugegraph.util.Bytes; +import com.baidu.hugegraph.util.E; + +public class PageState { + + private final byte[] position; + private final int offset; + + public PageState(byte[] position, int offset) { + E.checkNotNull(position, "position"); + this.position = position; + this.offset = offset; + } + + public byte[] position() { + return this.position; + } + + public int offset() { + return this.offset; + } + + @Override + public String toString() { + return Base64.getEncoder().encodeToString(this.toBytes()); + } + + public byte[] toBytes() { + int length = 2 + this.position.length + BytesBuffer.INT_LEN; + BytesBuffer buffer = BytesBuffer.allocate(length); + buffer.writeBytes(this.position); + buffer.writeInt(this.offset); + return buffer.bytes(); + } + + public static PageState fromString(String page) { + byte[] bytes; + try { + bytes = Base64.getDecoder().decode(page); + } catch (Exception e) { + throw new BackendException("Invalid page: '%s'", e, page); + } + return fromBytes(bytes); + } + + public static PageState fromBytes(byte[] bytes) { + if (bytes.length == 0) { + // The first page + return new PageState(new byte[0], 0); + } + try { + BytesBuffer buffer = BytesBuffer.wrap(bytes); + return new PageState(buffer.readBytes(), buffer.readInt()); + } catch (Exception e) { + throw new BackendException("Invalid page: '0x%s'", + e, Bytes.toHex(bytes)); + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java index fcb95595a6..2407a30fe0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java @@ -97,8 +97,11 @@ protected final long fetched() { } protected final void checkCapacity() throws LimitExceedException { - // Stop if reach capacity - this.query.checkCapacity(this.count); + // TODO: delete if + if (this.query.resultType().isGraph()) { + // Stop if reach capacity + this.query.checkCapacity(this.count); + } } protected final boolean exceedLimit() { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIds.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIds.java new file mode 100644 index 0000000000..d15dae6481 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIds.java @@ -0,0 +1,46 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.Collection; +import java.util.Set; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.util.E; + +public class EntireIds implements Ids { + + private final Set ids; + + public EntireIds(Collection ids) { + E.checkArgument(ids instanceof Set, + "The ids of EntireIds must be Set, but got '%s'", + ids.getClass().getName()); + this.ids = (Set) ids; + } + + public EntireIds(Set ids) { + this.ids = ids; + } + + public Set ids() { + return this.ids; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIndexIdsHolder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIndexIdsHolder.java new file mode 100644 index 0000000000..e8a50f202c --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/EntireIndexIdsHolder.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.Iterator; +import java.util.Set; +import java.util.function.Supplier; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.query.Query; +import com.baidu.hugegraph.util.E; + +public class EntireIndexIdsHolder extends IndexIdsHolder { + + private Set ids; + + public EntireIndexIdsHolder(Query query, Supplier idsFetcher) { + super(query, idsFetcher); + this.ids = null; + } + + public EntireIndexIdsHolder(Set ids) { + super(null, null); + this.ids = ids; + } + + public void merge(EntireIndexIdsHolder holder) { + this.all().addAll(holder.all()); + } + + public Set all() { + if (this.ids == null) { + Ids result = this.idsFetcher.get(); + E.checkState(result instanceof EntireIds, + "The result ids must be EntireIds for " + + "EntireIndexIdsHolder"); + this.ids = ((EntireIds) result).ids(); + assert this.ids != null; + } + return this.ids; + } + + @Override + public Iterator iterator() { + return this.all().iterator(); + } + + @Override + public long size() { + return this.all().size(); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java index d7daf54ff0..04eda7d88f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java @@ -44,12 +44,12 @@ import com.baidu.hugegraph.backend.query.Condition.Relation; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.query.ConditionQueryFlatten; -import com.baidu.hugegraph.backend.query.IdQuery; import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.AbstractSerializer; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.exception.NoIndexException; +import com.baidu.hugegraph.iterator.Metadatable; import com.baidu.hugegraph.job.EphemeralJob; import com.baidu.hugegraph.job.EphemeralJobBuilder; import com.baidu.hugegraph.perf.PerfUtil.Watched; @@ -76,7 +76,6 @@ public class GraphIndexTransaction extends AbstractTransaction { private static final String INDEX_EMPTY_SYM = "\u0000"; - private static final Query EMPTY_QUERY = new ConditionQuery(null); private final Analyzer textAnalyzer; @@ -226,7 +225,7 @@ private void updateIndex(IndexLabel indexLabel, Object propValue, * @return converted id query */ @Watched(prefix = "index") - public Query indexQuery(ConditionQuery query) { + public IndexIdsHolderChain indexQuery(ConditionQuery query) { // Index query must have been flattened in Graph tx query.checkFlattened(); @@ -245,26 +244,18 @@ public Query indexQuery(ConditionQuery query) { // Query by index query.optimized(OptimizedType.INDEX.ordinal()); - Set ids; if (query.allSysprop() && conds.size() == 1 && query.containsCondition(HugeKeys.LABEL)) { // Query only by label - ids = this.queryByLabel(query); + return this.queryByLabel(query); } else { // Query by userprops (or userprops + label) - ids = this.queryByUserprop(query); + return this.queryByUserprop(query); } - - if (ids.isEmpty()) { - return EMPTY_QUERY; - } - - // Wrap id(s) by IdQuery - return new IdQuery(query, ids); } @Watched(prefix = "index") - private Set queryByLabel(ConditionQuery query) { + private IndexIdsHolderChain queryByLabel(ConditionQuery query) { HugeType queryType = query.resultType(); IndexLabel il = IndexLabel.label(queryType); Id label = (Id) query.condition(HugeKeys.LABEL); @@ -290,15 +281,22 @@ private Set queryByLabel(ConditionQuery query) { indexQuery.eq(HugeKeys.INDEX_LABEL_ID, il.id()); indexQuery.eq(HugeKeys.FIELD_VALUES, label); // Set offset and limit to avoid redundant element ids + indexQuery.page(query.page()); indexQuery.limit(query.limit()); indexQuery.offset(query.offset()); indexQuery.capacity(query.capacity()); - return this.doIndexQuery(il, indexQuery); + IndexIdsHolder idsHolder; + if (query.paging()) { + idsHolder = this.doIndexQueryInPage(il, indexQuery); + } else { + idsHolder = this.doIndexQuery(il, indexQuery); + } + return new IndexIdsHolderChain(idsHolder); } @Watched(prefix = "index") - private Set queryByUserprop(ConditionQuery query) { + private IndexIdsHolderChain queryByUserprop(ConditionQuery query) { // Get user applied label or collect all qualified labels with // related index labels Set indexes = this.collectMatchedIndexes(query); @@ -309,31 +307,114 @@ private Set queryByUserprop(ConditionQuery query) { // Value type of Condition not matched if (!validQueryConditionValues(this.graph(), query)) { - return ImmutableSet.of(); + return new IndexIdsHolderChain(IndexIdsHolder.EMPTY); } // Do index query - Set ids = InsertionOrderUtil.newSet(); + IndexIdsHolderChain chain = new IndexIdsHolderChain(query.paging()); + long count = 0; for (MatchedIndex index : indexes) { if (index.containsSearchIndex()) { + ConditionQuery searchQuery = this.constructSearchQuery(query, index); // Do search-index query - ids.addAll(this.queryByUserpropWithSearchIndex(query, index)); + if (query.paging()) { + chain.link(this.doSearchIndexInPage(searchQuery, index)); + } else { + chain.link(this.doSearchIndex(searchQuery, index)); + } } else { // Do secondary-index or range-index query IndexQueries queries = index.constructIndexQueries(query); - ids.addAll(this.intersectIndexQueries(queries)); + IndexIdsHolder idsHolder = this.intersectIndexQueries(queries); + + count += idsHolder.size(); + chain.link(idsHolder); } - if (query.reachLimit(ids.size())) { + if (query.reachLimit(count)) { + break; + } + } + return chain; +// return limit(ids, query); + } + + private List doSearchIndexInPage(ConditionQuery query, + MatchedIndex index) { + // Do query + List idsHolders = new ArrayList<>(); + long count = 0; + for (ConditionQuery q : ConditionQueryFlatten.flatten(query)) { + IndexQueries queries = index.constructIndexQueries(q); + IndexIdsHolder idsHolder = this.intersectIndexQueries(queries); + + count += idsHolder.size(); + idsHolders.add(idsHolder); + + if (query.reachLimit(count)) { break; } } - return limit(ids, query); + return idsHolders; + } + + @SuppressWarnings("unchecked") + private IndexIdsHolder doSearchIndex(ConditionQuery query, + MatchedIndex index) { + // Do query + Set ids = InsertionOrderUtil.newSet(); + EntireIndexIdsHolder idsHolder = new EntireIndexIdsHolder(ids); + for (ConditionQuery q : ConditionQueryFlatten.flatten(query)) { + IndexQueries queries = index.constructIndexQueries(q); + IndexIdsHolder holder = this.intersectIndexQueries(queries); + assert holder instanceof EntireIndexIdsHolder; + idsHolder.merge((EntireIndexIdsHolder) holder); + } + return idsHolder; } @Watched(prefix = "index") - private Set queryByUserpropWithSearchIndex(ConditionQuery query, - MatchedIndex index) { + private IndexIdsHolder intersectIndexQueries(IndexQueries queries) { + if (queries.size() == 1) { + return this.doSingleOrCompositeIndex(queries); + } else { + return this.doJointIndex(queries); + } + } + + @Watched(prefix = "index") + private IndexIdsHolder doSingleOrCompositeIndex(IndexQueries queries) { + assert queries.size() == 1; + Map.Entry e = queries.entrySet() + .iterator().next(); + IndexLabel indexLabel = e.getKey(); + ConditionQuery query = e.getValue(); + if (query.paging()) { + return this.doIndexQueryInPage(indexLabel, query); + } else { + return this.doIndexQuery(indexLabel, query); + } + } + + @Watched(prefix = "index") + private IndexIdsHolder doJointIndex(IndexQueries queries) { + Set intersectIds = null; + for (Map.Entry e : queries.entrySet()) { + Set ids = this.doIndexQuery(e.getKey(), e.getValue()).all(); + if (intersectIds == null) { + intersectIds = ids; + } else { + CollectionUtil.intersectWithModify(intersectIds, ids); + } + if (intersectIds.isEmpty()) { + break; + } + } + return new EntireIndexIdsHolder(intersectIds); + } + + private ConditionQuery constructSearchQuery(ConditionQuery query, + MatchedIndex index) { ConditionQuery originQuery = query; Set indexFields = new HashSet<>(); // Convert has(key, text) to has(key, textContainsAny(word1, word2)) @@ -371,14 +452,7 @@ private Set queryByUserpropWithSearchIndex(ConditionQuery query, } return true; }); - - // Do query - Set ids = InsertionOrderUtil.newSet(); - for (ConditionQuery q : ConditionQueryFlatten.flatten(query)) { - IndexQueries queries = index.constructIndexQueries(q); - ids.addAll(this.intersectIndexQueries(queries)); - } - return ids; + return query; } private boolean matchSearchIndexWords(String propValue, String fieldValue) { @@ -396,44 +470,60 @@ private boolean needIndexForLabel() { } @Watched(prefix = "index") - private Collection intersectIndexQueries(IndexQueries queries) { - Collection intersectIds = null; + private EntireIndexIdsHolder doIndexQuery(IndexLabel indexLabel, + ConditionQuery query) { + LockUtil.Locks locks = new LockUtil.Locks(this.graph().name()); + try { + locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); + locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id()); - for (Map.Entry entry : queries.entrySet()) { - Set ids = this.doIndexQuery(entry.getKey(), entry.getValue()); - if (intersectIds == null) { - intersectIds = ids; - } else { - CollectionUtil.intersectWithModify(intersectIds, ids); - } - if (intersectIds.isEmpty()) { - break; - } + return new EntireIndexIdsHolder(query, () -> { + Collection ids = InsertionOrderUtil.newSet(); + Iterator entries = super.query(query); + while(entries.hasNext()) { + HugeIndex index = this.serializer.readIndex(graph(), query, + entries.next()); + ids.addAll(index.elementIds()); + if (query.reachLimit(ids.size())) { + break; + } + } + return new EntireIds(ids); + }); + } finally { + locks.unlock(); } - return intersectIds; } @Watched(prefix = "index") - private Set doIndexQuery(IndexLabel indexLabel, ConditionQuery query) { - Set ids = InsertionOrderUtil.newSet(); + private PagedIndexIdsHolder doIndexQueryInPage(IndexLabel indexLabel, + ConditionQuery query) { LockUtil.Locks locks = new LockUtil.Locks(this.graph().name()); try { locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id()); locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id()); - Iterator entries = super.query(query); - while(entries.hasNext()) { - HugeIndex index = this.serializer.readIndex(graph(), query, - entries.next()); - ids.addAll(index.elementIds()); - if (query.reachLimit(ids.size())) { - break; + return new PagedIndexIdsHolder(query, () -> { + Collection ids = InsertionOrderUtil.newList(); + Iterator entries = super.query(query); + while(entries.hasNext()) { + HugeIndex index = this.serializer.readIndex(graph(), query, + entries.next()); + ids.addAll(index.elementIds()); + if (query.reachLimit(ids.size())) { + break; + } } - } + E.checkState(entries instanceof Metadatable, + "The entries must be Metadatable when query " + + "in paging, but got '%s'", + entries.getClass().getName()); + String page = (String) ((Metadatable) entries).metadata("page"); + return new PagedIds(ids, page); + }); } finally { locks.unlock(); } - return ids; } @Watched(prefix = "index") diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index f4b3e4c173..2e864eab8d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -305,14 +305,30 @@ public void rollback() throws BackendException { } } + public IndexIdsHolderChain indexQuery(ConditionQuery query) { + /* + * Optimize by index-query + * It will return a list of id (maybe empty) if success, + * or throw exception if there is no any index for query properties. + */ + this.beforeRead(); + try { + return this.indexTx.indexQuery(query); + } finally { + this.afterRead(); + } + } + @Override public Iterator query(Query query) { if (!(query instanceof ConditionQuery)) { return super.query(query); } + boolean paging = query.paging(); + IndexIdsHolderChain chain = new IndexIdsHolderChain(query.paging()); List queries = new ArrayList<>(); - IdQuery ids = new IdQuery(query.resultType(), query); + for (ConditionQuery cq: ConditionQueryFlatten.flatten( (ConditionQuery) query)) { Query q = this.optimizeQuery(cq); @@ -321,18 +337,37 @@ public Iterator query(Query query) { * 1.sysprop-query, which would not be empty. * 2.index-query result(ids after optimization), which may be empty. */ - if (q.getClass() == IdQuery.class && !q.ids().isEmpty()) { - ids.query(q.ids()); + if (q == null) { + chain.link(this.indexQuery(cq)); } else if (!q.empty()) { // Return empty if there is no result after index-query queries.add(q); } } + Iterator idsHolders = chain.toIterator(); ExtendableIterator rs = new ExtendableIterator<>(); - if (!ids.empty()) { - queries.add(ids); + if (paging) { + idsHolders.forEachRemaining(idsHolder -> { + rs.extend(new MapperIterator<>(idsHolder.iterator(), (id) -> { + IdQuery idQuery = new IdQuery(query.resultType(), id); + Iterator results = super.query(idQuery); + if (results.hasNext()) { + return results.next(); + } else { + return null; + } + })); + }); + } else { + idsHolders.forEachRemaining(idsHolder -> { + Set ids = ((EntireIndexIdsHolder) idsHolder).all(); + if (!ids.isEmpty()) { + queries.add(new IdQuery(query, ids)); + } + }); } + for (Query q : queries) { rs.extend(super.query(q)); } @@ -934,7 +969,7 @@ public static void verifyEdgesConditionQuery(ConditionQuery query) { } } - protected Query optimizeQuery(ConditionQuery query) { + private Query optimizeQuery(ConditionQuery query) { Id label = (Id) query.condition(HugeKeys.LABEL); // Optimize vertex query @@ -1018,18 +1053,8 @@ protected Query optimizeQuery(ConditionQuery query) { return query; } } - - /* - * Optimize by index-query - * It will return a list of id (maybe empty) if success, - * or throw exception if there is no any index for query properties. - */ - this.beforeRead(); - try { - return this.indexTx.indexQuery(query); - } finally { - this.afterRead(); - } + // TODO: any better value? + return null; } private VertexLabel checkVertexLabel(Object label, boolean verifyLabel) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/Ids.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/Ids.java new file mode 100644 index 0000000000..a8028057ed --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/Ids.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.Collection; + +import com.baidu.hugegraph.backend.id.Id; + +public interface Ids { + + public Collection ids(); +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolder.java new file mode 100644 index 0000000000..43dcec0531 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolder.java @@ -0,0 +1,52 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.Iterator; +import java.util.function.Supplier; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.query.Query; + +public abstract class IndexIdsHolder implements Iterable { + + protected final Query query; + protected final Supplier idsFetcher; + + public IndexIdsHolder(Query query, Supplier idsFetcher) { + this.query = query; + this.idsFetcher = idsFetcher; + } + + public abstract long size(); + + public static final IndexIdsHolder EMPTY = new IndexIdsHolder(null, null) { + + @Override + public long size() { + return 0; + } + + @Override + public Iterator iterator() { + return null; + } + }; +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolderChain.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolderChain.java new file mode 100644 index 0000000000..81e58f70b9 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexIdsHolderChain.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import com.baidu.hugegraph.util.E; + +public final class IndexIdsHolderChain { + + // Just as an flag + private final boolean paged; + private final List idsHolders; + + public IndexIdsHolderChain(boolean paged) { + this.paged = paged; + this.idsHolders = new ArrayList<>(); + } + + public boolean paged() { + return this.paged; + } + + public List idsHolders() { + return this.idsHolders; + } + + public IndexIdsHolderChain(IndexIdsHolder idsHolder) { + if (idsHolder instanceof PagedIndexIdsHolder) { + this.paged = true; + } else { + assert idsHolder instanceof EntireIndexIdsHolder; + this.paged = false; + } + this.idsHolders = new ArrayList<>(); + this.idsHolders.add(idsHolder); + } + + public void link(IndexIdsHolder idsHolder) { + this.checkIdsHolderType(idsHolder); + if (this.paged) { + this.idsHolders.add(idsHolder); + } else { + if (this.idsHolders.isEmpty()) { + this.idsHolders.add(idsHolder); + } else { + IndexIdsHolder selfIdsHolder = this.idsHolders.get(0); + assert selfIdsHolder instanceof EntireIndexIdsHolder; + EntireIndexIdsHolder holder = (EntireIndexIdsHolder) idsHolder; + ((EntireIndexIdsHolder) selfIdsHolder).merge(holder); + } + } + } + + public void link(List idsHolders) { + for (IndexIdsHolder idsHolder : idsHolders) { + this.link(idsHolder); + } + } + + public void link(IndexIdsHolderChain chain) { + E.checkArgument((this.paged && chain.paged) || + (!this.paged && !chain.paged), + "Only same IndexIdsHolderChain can be linked"); + this.link(chain.idsHolders()); + } + + private void checkIdsHolderType(IndexIdsHolder idsHolder) { + if (this.paged) { + E.checkArgument(idsHolder instanceof PagedIndexIdsHolder, + "The IndexIdsHolder to be linked must be " + + "PagedIndexIdsHolder in paged mode"); + } else { + E.checkArgument(idsHolder instanceof EntireIndexIdsHolder, + "The IndexIdsHolder to be linked must be " + + "EntireIndexIdsHolder in non-paged mode"); + } + } + + public Iterator toIterator() { + return this.idsHolders.iterator(); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIds.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIds.java new file mode 100644 index 0000000000..085eabd86b --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIds.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.Collection; +import java.util.List; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.util.E; + +public final class PagedIds implements Ids { + + private final List ids; + private final String page; + + public PagedIds(Collection ids, String page) { + E.checkArgument(ids instanceof List, + "The ids of PagedIds must be List, but got '%s'", + ids.getClass().getName()); + this.ids = (List) ids; + this.page = page; + } + + public PagedIds(List ids, String page) { + this.ids = ids; + this.page = page; + } + + public List ids() { + return this.ids; + } + + public String page() { + return this.page; + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIndexIdsHolder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIndexIdsHolder.java new file mode 100644 index 0000000000..ce1e84a383 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/PagedIndexIdsHolder.java @@ -0,0 +1,113 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.tx; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Supplier; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.query.Query; +import com.baidu.hugegraph.util.E; + +public final class PagedIndexIdsHolder extends IndexIdsHolder { + + private List ids; + private String page; + + public PagedIndexIdsHolder(Query query, Supplier idsFetcher) { + super(query, idsFetcher); + E.checkState(query.paging(), + "query '%s' must carry the page in pagination mode", + query); + this.ids = null; + this.resetPage(); + } + + private void resetPage() { + this.page = ""; + } + + public String page() { + return this.page; + } + + @Override + public Iterator iterator() { + return new IdsIterator(); + } + + @Override + public long size() { + if (this.ids == null) { + return 0; + } + return this.ids.size(); + } + + public class IdsIterator implements Iterator { + + private int cursor; + private boolean finished; + + private IdsIterator() { + this.cursor = 0; + this.finished = false; + } + + @Override + public boolean hasNext() { + // When local data is empty or consumed, then fetch from the backend + if (ids == null || this.cursor >= ids.size()) { + this.fetch(); + } + assert ids != null; + return this.cursor < ids.size(); + } + + private void fetch() { + if (this.finished) { + return; + } + + query.page(page); + Ids result = idsFetcher.get(); + E.checkState(result instanceof PagedIds, + "The result ids must be PagedIds for " + + "PagedIndexIdsHolder"); + ids = ((PagedIds) result).ids(); + page = ((PagedIds) result).page(); + this.cursor = 0; + + if (ids.size() != query.limit() || page == null) { + this.finished = true; + } + } + + @Override + public Id next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + return ids.get(this.cursor++); + } + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 1843e3a53f..91afb5ffe2 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -206,6 +206,14 @@ public static synchronized CoreOptions instance() { (60 * 10) ); + public static final ConfigOption PAGE_SIZE = + new ConfigOption<>( + "page_size", + "The size of each page when paging.", + rangeInt(0, 10000), + (1) + ); + public static final ConfigOption SNOWFLAKE_WORKER_ID = new ConfigOption<>( "snowflake.worker_id", diff --git a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java index aa2a958091..556a939674 100644 --- a/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java +++ b/hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example2.java @@ -19,20 +19,16 @@ package com.baidu.hugegraph.example; -import java.util.List; - import org.apache.tinkerpop.gremlin.process.traversal.P; -import org.apache.tinkerpop.gremlin.process.traversal.Path; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; -import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.slf4j.Logger; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.schema.SchemaManager; +import com.baidu.hugegraph.traversal.optimize.Text; import com.baidu.hugegraph.util.Log; public class Example2 { @@ -53,90 +49,39 @@ public static void main(String[] args) throws InterruptedException { } public static void traversal(final HugeGraph graph) { - GraphTraversalSource g = graph.traversal(); - - GraphTraversal vertexs = g.V(); - System.out.println(">>>> query all vertices: size=" + - vertexs.toList().size()); - - List edges = g.E().toList(); - System.out.println(">>>> query all edges: size=" + - edges.size()); - - List names = g.V().inE("knows").limit(2) - .outV().values("name").toList(); - System.out.println(">>>> query vertex(with props) of edges: " + names); - assert names.size() == 2 : names.size(); - - names = g.V().as("a") - .out("knows") - .and() - .out("created").in("created").as("a").values("name") - .toList(); - System.out.println(">>>> query with AND: " + names); - assert names.size() == 1 : names.size(); - - List vertex = g.V().has("age", 29).toList(); - System.out.println(">>>> age = 29: " + vertex); - assert vertex.size() == 1 && - vertex.get(0).value("name").equals("marko"); - - vertex = g.V() - .has("age", 29) - .has("city", "Beijing") - .toList(); - System.out.println(">>>> age = 29 and city is Beijing: " + vertex); - assert vertex.size() == 1 && - vertex.get(0).value("name").equals("marko"); - - edges = g.E().has("weight", P.lt(1.0)).toList(); - System.out.println(">>>> edges with weight < 1.0: " + edges); - assert edges.size() == 4; - - String person = graph.schema().getVertexLabel("person").id().asString(); - String software = graph.schema().getVertexLabel("software").id() - .asString(); - String markoId = String.format("%s:%s", person, "marko"); - String joshId = String.format("%s:%s", person, "josh"); - String lopId = String.format("%s:%s", software, "lop"); - - vertex = g.V(joshId) - .bothE("created") - .has("weight", P.lt(1.0)) - .otherV() - .toList(); - System.out.println(">>>> josh's both edges with weight < 1.0: " + - vertex); - assert vertex.size() == 1 && vertex.get(0).value("name").equals("lop"); - - List paths = g.V(markoId).out().out().path().by("name").toList(); - System.out.println(">>>> test out path: " + paths); - assert paths.size() == 2; - assert paths.get(0).get(0).equals("marko"); - assert paths.get(0).get(1).equals("josh"); - assert paths.get(0).get(2).equals("lop"); - assert paths.get(1).get(0).equals("marko"); - assert paths.get(1).get(1).equals("josh"); - assert paths.get(1).get(2).equals("ripple"); - - paths = shortestPath(graph, markoId, lopId, 5); - System.out.println(">>>> test shortest path: " + paths.get(0)); - assert paths.get(0).get(0).equals("marko"); - assert paths.get(0).get(1).equals("lop"); - } - - public static List shortestPath(final HugeGraph graph, - Object from, Object to, - int maxDepth) { - GraphTraversal t = graph.traversal() - .V(from) - .repeat(__.out().simplePath()) - .until(__.hasId(to).or().loops().is(P.gt(maxDepth))) - .hasId(to) - .path().by("name") - .limit(1); - return t.toList(); + GraphTraversal traversal; + + // Query total vertices + System.out.println(g.V().hasLabel("person").toList()); + + // Query by label, DONE + System.out.println(">>>> label = person vertices: " + g.V().hasLabel("person").has("~page", "").limit(2).toList()); + System.out.println(">>>> label = knows edges: " + g.E().hasLabel("knows").has("~page", "").limit(1).toList()); + + // Query by single secondary index + // 不分页，显示全部 +// System.out.println(">>>> name = marko vertices:"); +// System.out.println(traversal.has("name", "marko").toList()); + + // 单索引secondary + System.out.println(">>>> name = marko vertices:"); + traversal = g.V().has("name", "marko").has("~page", "").limit(3); + while (traversal.hasNext()) { + System.out.println(traversal.next()); + } + // 单索引range + System.out.println(">>>> price >= 100 && price < 300 vertices:"); + traversal = g.V().has("price", P.between(100, 300)).has("~page", "").limit(2); + while (traversal.hasNext()) { + System.out.println(traversal.next()); + } + // 单索引search + System.out.println(">>>> city contains Haidian vertices:"); + traversal = g.V().has("city", Text.contains("Beijing Haidian")).has("~page", "").limit(2); + while (traversal.hasNext()) { + System.out.println(traversal.next()); + } } public static void load(final HugeGraph graph) { @@ -152,29 +97,35 @@ public static void load(final HugeGraph graph) { schema.vertexLabel("person") .properties("name", "age", "city") - .primaryKeys("name") + .useCustomizeStringId() .nullableKeys("age") .ifNotExist() .create(); schema.vertexLabel("software") .properties("name", "lang", "price") - .primaryKeys("name") + .useCustomizeStringId() .nullableKeys("price") .ifNotExist() .create(); - schema.indexLabel("personByCity") + schema.indexLabel("personByNameAndCity") .onV("person") - .by("city") + .by("name", "city") .secondary() .ifNotExist() .create(); - schema.indexLabel("personByAgeAndCity") + schema.indexLabel("softwareByNameAndLang") + .onV("software") + .by("name", "lang") + .ifNotExist() + .create(); + + schema.indexLabel("personByCity") .onV("person") - .by("age", "city") - .secondary() + .search() + .by("city") .ifNotExist() .create(); @@ -202,48 +153,52 @@ public static void load(final HugeGraph graph) { .ifNotExist() .create(); - schema.indexLabel("createdByDate") - .onE("created") - .by("date") - .secondary() - .ifNotExist() - .create(); - - schema.indexLabel("createdByWeight") - .onE("created") - .by("weight") - .range() - .ifNotExist() - .create(); - - schema.indexLabel("knowsByWeight") - .onE("knows") - .by("weight") - .range() - .ifNotExist() - .create(); +// schema.indexLabel("createdByDate") +// .onE("created") +// .by("date") +// .secondary() +// .ifNotExist() +// .create(); +// +// schema.indexLabel("createdByWeight") +// .onE("created") +// .by("weight") +// .range() +// .ifNotExist() +// .create(); +// +// schema.indexLabel("knowsByWeight") +// .onE("knows") +// .by("weight") +// .range() +// .ifNotExist() +// .create(); graph.tx().open(); - Vertex marko = graph.addVertex(T.label, "person", "name", "marko", - "age", 29, "city", "Beijing"); - Vertex vadas = graph.addVertex(T.label, "person", "name", "vadas", - "age", 27, "city", "Hongkong"); - Vertex lop = graph.addVertex(T.label, "software", "name", "lop", - "lang", "java", "price", 328); - Vertex josh = graph.addVertex(T.label, "person", "name", "josh", - "age", 32, "city", "Beijing"); - Vertex ripple = graph.addVertex(T.label, "software", "name", "ripple", - "lang", "java", "price", 199); - Vertex peter = graph.addVertex(T.label, "person", "name", "peter", - "age", 35, "city", "Shanghai"); - - marko.addEdge("knows", vadas, "date", "20160110", "weight", 0.5); - marko.addEdge("knows", josh, "date", "20130220", "weight", 1.0); - marko.addEdge("created", lop, "date", "20171210", "weight", 0.4); - josh.addEdge("created", lop, "date", "20091111", "weight", 0.4); + Vertex p_marko = graph.addVertex(T.label, "person", T.id, "p_marko", "name", "marko", "age", 29, "city", "Beijing Haidian"); + graph.tx().commit(); + Vertex p_marko2 = graph.addVertex(T.label, "person", T.id, "p_marko2", "name", "marko", "age", 29, "city", "Beijing Chaoyang"); + graph.tx().commit(); + Vertex vadas = graph.addVertex(T.label, "person", T.id, "vadas", "name", "vadas", "age", 27, "city", "Beijing Haidian"); + graph.tx().commit(); + Vertex josh = graph.addVertex(T.label, "person", T.id, "josh", "name", "josh", "age", 32, "city", "Beijing"); + graph.tx().commit(); + Vertex peter = graph.addVertex(T.label, "person", T.id, "peter", "name", "peter", "age", 35, "city", "Shanghai"); + graph.tx().commit(); + Vertex s_marko = graph.addVertex(T.label, "software", T.id, "s_marko", "name", "marko", "lang", "java", "price", 100); + graph.tx().commit(); + Vertex ripple = graph.addVertex(T.label, "software", T.id, "ripple", "name", "ripple", "lang", "java", "price", 200); + graph.tx().commit(); + Vertex hadoop = graph.addVertex(T.label, "software", T.id, "hadoop", "name", "hadoop", "lang", "java", "price", 300); + graph.tx().commit(); + + p_marko.addEdge("knows", vadas, "date", "20160110", "weight", 0.5); + p_marko.addEdge("knows", josh, "date", "20130220", "weight", 1.0); + p_marko.addEdge("created", s_marko, "date", "20171210", "weight", 0.4); + josh.addEdge("created", s_marko, "date", "20091111", "weight", 0.4); josh.addEdge("created", ripple, "date", "20171210", "weight", 1.0); - peter.addEdge("created", lop, "date", "20170324", "weight", 0.2); + peter.addEdge("created", s_marko, "date", "20170324", "weight", 0.2); graph.tx().commit(); } diff --git a/hugegraph-example/src/main/resources/hugegraph.properties b/hugegraph-example/src/main/resources/hugegraph.properties index fa596fb7f5..0dfc92aecd 100644 --- a/hugegraph-example/src/main/resources/hugegraph.properties +++ b/hugegraph-example/src/main/resources/hugegraph.properties @@ -1,10 +1,7 @@ gremlin.graph=com.baidu.hugegraph.HugeFactory -backend=cassandra -serializer=cassandra - -#backend=rocksdb -#serializer=binary +backend=rocksdb +serializer=binary store=hugegraph rate_limit=0 diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java index 43b1e5e1f6..34e34be5b8 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java @@ -40,7 +40,7 @@ import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry; import com.baidu.hugegraph.backend.serializer.BinaryEntryIterator; -import com.baidu.hugegraph.backend.serializer.BinaryEntryIterator.PageState; +import com.baidu.hugegraph.backend.serializer.PageState; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn; import com.baidu.hugegraph.backend.store.BackendEntryIterator; diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java index 795dfe9d87..381a20ed1b 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -31,7 +31,7 @@ import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry; import com.baidu.hugegraph.backend.serializer.BinaryEntryIterator; -import com.baidu.hugegraph.backend.serializer.BinaryEntryIterator.PageState; +import com.baidu.hugegraph.backend.serializer.PageState; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn; import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumnIterator; diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index 9fad0b6d3a..9214b4126d 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -2896,7 +2896,7 @@ public void testScanVertex() { long splitSize = 1 * 1024 * 1024; Object splits = graph.graphTransaction() - .metadata(HugeType.VERTEX, "splits", splitSize); + .metadata(HugeType.VERTEX, "splits", splitSize); for (Shard split : (List) splits) { ConditionQuery q = new ConditionQuery(HugeType.VERTEX); q.scan(split.start(), split.end()); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java index bd527ddf0a..e276dd4ea5 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java @@ -642,7 +642,7 @@ public void testRemoveVertexLabelWithVertex() { graph().tx().commit(); List vertex = graph().traversal().V().hasLabel("person") - .toList(); + .toList(); Assert.assertNotNull(vertex); Assert.assertEquals(3, vertex.size()); @@ -679,7 +679,7 @@ public void testRemoveVertexLabelWithVertexAndRangeIndex() { graph().tx().commit(); List vertex = graph().traversal().V().hasLabel("person") - .has("age", P.inside(4, 10)).toList(); + .has("age", P.inside(4, 10)).toList(); Assert.assertNotNull(vertex); Assert.assertEquals(2, vertex.size()); diff --git a/hugegraph-test/src/main/resources/hugegraph.properties b/hugegraph-test/src/main/resources/hugegraph.properties index 54d5b14850..796528b8a5 100644 --- a/hugegraph-test/src/main/resources/hugegraph.properties +++ b/hugegraph-test/src/main/resources/hugegraph.properties @@ -1,7 +1,7 @@ gremlin.graph=com.baidu.hugegraph.HugeFactory -backend=${backend} -serializer=${serializer} +backend=rocksdb +serializer=binary store=hugegraph