From c0617e1af2d969aa17ab9541986e2748d5849ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Thu, 19 Oct 2023 11:13:21 +0200 Subject: [PATCH] GH-4819 initial support for specifying statement order from the SailConnection through to the ExtensibleStore datastructure, without support for inferred statements, ReadCommitted or the caching layers --- core/common/iterator/pom.xml | 5 + .../common/iteration/DualUnionIteration.java | 21 ++++ core/common/ordering/pom.xml | 20 ++++ .../rdf4j/common/ordering/StatementOrder.java | 21 ++++ core/common/pom.xml | 1 + core/sail/api/pom.xml | 5 + .../eclipse/rdf4j/sail/SailConnection.java | 30 +++++ .../sail/helpers/AbstractSailConnection.java | 29 +++++ .../eclipse/rdf4j/sail/base/SailDataset.java | 32 +++++ .../rdf4j/sail/base/SailSourceConnection.java | 11 ++ .../rdf4j/sail/base/UnionSailDataset.java | 54 +++++++++ .../DataStructureInterface.java | 23 +++- .../sail/extensiblestore/EagerReadCache.java | 7 ++ .../extensiblestore/ExtensibleSailSource.java | 17 +++ .../sail/extensiblestore/LazyReadCache.java | 7 ++ .../extensiblestore/ReadCommittedWrapper.java | 7 ++ .../sail/extensiblestore/SortedIteration.java | 90 ++++++++++++++ .../EvaluationStatisticsWrapper.java | 20 ++++ ...bleStoreConnectionOrderedImplForTests.java | 20 ++++ .../ExtensibleStoreOrderedImplForTests.java | 50 ++++++++ .../ordered/OrderedDataStructure.java | 110 ++++++++++++++++++ .../extensiblestore/ordered/OrderedTest.java | 97 +++++++++++++++ 22 files changed, 676 insertions(+), 1 deletion(-) create mode 100644 core/common/ordering/pom.xml create mode 100644 core/common/ordering/src/main/java/org/eclipse/rdf4j/common/ordering/StatementOrder.java create mode 100644 core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/SortedIteration.java create mode 100644 core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreConnectionOrderedImplForTests.java create mode 100644 core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreOrderedImplForTests.java create mode 100644 core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedDataStructure.java create mode 100644 core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedTest.java diff --git a/core/common/iterator/pom.xml b/core/common/iterator/pom.xml index c2e21734002..a6a276ee30e 100644 --- a/core/common/iterator/pom.xml +++ b/core/common/iterator/pom.xml @@ -15,6 +15,11 @@ rdf4j-model-api ${project.version} + + ${project.groupId} + rdf4j-common-ordering + ${project.version} + org.slf4j slf4j-api diff --git a/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DualUnionIteration.java b/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DualUnionIteration.java index e581e85a59a..0b1a7de9766 100644 --- a/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DualUnionIteration.java +++ b/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DualUnionIteration.java @@ -11,8 +11,11 @@ package org.eclipse.rdf4j.common.iteration; +import java.util.Comparator; import java.util.NoSuchElementException; +import org.eclipse.rdf4j.common.ordering.StatementOrder; + /** * Provides a bag union of the two provided iterations. */ @@ -32,6 +35,12 @@ private DualUnionIteration(CloseableIteration iteration1, this.iteration2 = iteration2; } + public DualUnionIteration(Comparator cmp, CloseableIteration leftIteration, + CloseableIteration rightIteration) { + throw new UnsupportedOperationException("Not implemented yet"); + + } + public static CloseableIteration getWildcardInstance( CloseableIteration leftIteration, CloseableIteration rightIteration) { @@ -44,6 +53,18 @@ public static CloseableIteration getWildca } } + public static CloseableIteration getWildcardInstance(Comparator cmp, + CloseableIteration leftIteration, CloseableIteration rightIteration) { + + if (rightIteration instanceof EmptyIteration) { + return leftIteration; + } else if (leftIteration instanceof EmptyIteration) { + return rightIteration; + } else { + return new DualUnionIteration<>(cmp, leftIteration, rightIteration); + } + } + public static CloseableIteration getInstance(CloseableIteration leftIteration, CloseableIteration rightIteration) { diff --git a/core/common/ordering/pom.xml b/core/common/ordering/pom.xml new file mode 100644 index 00000000000..02ac6d98f4e --- /dev/null +++ b/core/common/ordering/pom.xml @@ -0,0 +1,20 @@ + + + 4.0.0 + + org.eclipse.rdf4j + rdf4j-common + 5.0.0-SNAPSHOT + + rdf4j-common-ordering + RDF4J: common ordering + RDF4J common ordering classes + + + + + maven-assembly-plugin + + + + diff --git a/core/common/ordering/src/main/java/org/eclipse/rdf4j/common/ordering/StatementOrder.java b/core/common/ordering/src/main/java/org/eclipse/rdf4j/common/ordering/StatementOrder.java new file mode 100644 index 00000000000..04ced490cec --- /dev/null +++ b/core/common/ordering/src/main/java/org/eclipse/rdf4j/common/ordering/StatementOrder.java @@ -0,0 +1,21 @@ +/******************************************************************************* + * Copyright (c) 2023 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +package org.eclipse.rdf4j.common.ordering; + +public enum StatementOrder { + + S, + P, + O, + C; + +} diff --git a/core/common/pom.xml b/core/common/pom.xml index f8aa4c5e689..8c26d4c8ddd 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -15,6 +15,7 @@ exception io iterator + ordering text transaction xml diff --git a/core/sail/api/pom.xml b/core/sail/api/pom.xml index 946090c1350..143fc9f1f74 100644 --- a/core/sail/api/pom.xml +++ b/core/sail/api/pom.xml @@ -30,6 +30,11 @@ rdf4j-common-transaction ${project.version} + + ${project.groupId} + rdf4j-common-ordering + ${project.version} + ${project.groupId} rdf4j-collection-factory-api diff --git a/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/SailConnection.java b/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/SailConnection.java index a5f428f4714..4c0f04ebc18 100644 --- a/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/SailConnection.java +++ b/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/SailConnection.java @@ -11,9 +11,11 @@ package org.eclipse.rdf4j.sail; import java.util.Optional; +import java.util.Set; import org.eclipse.rdf4j.common.annotation.Experimental; import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.common.transaction.IsolationLevel; import org.eclipse.rdf4j.common.transaction.TransactionSetting; import org.eclipse.rdf4j.model.IRI; @@ -118,6 +120,30 @@ CloseableIteration evaluate(TupleExpr tupleExpr, CloseableIteration getStatements(Resource subj, IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException; + /** + * Gets all statements from the specified contexts that have a specific subject, predicate and/or object. All three + * parameters may be null to indicate wildcards. The includeInferred parameter can be used to control + * which statements are fetched: all statements or only the statements that have been added explicitly. + * + * @param statementOrder The order that the statements should be returned in. + * @param subj A Resource specifying the subject, or null for a wildcard. + * @param pred A URI specifying the predicate, or null for a wildcard. + * @param obj A Value specifying the object, or null for a wildcard. + * @param includeInferred if false, no inferred statements are returned; if true, inferred statements are returned + * if available + * @param contexts The context(s) to get the data from. Note that this parameter is a vararg and as such is + * optional. If no contexts are specified the method operates on the entire repository. A + * null value can be used to match context-less statements. + * @return The statements matching the specified pattern. + * @throws SailException If the Sail object encountered an error or unexpected situation internally. + * @throws IllegalStateException If the connection has been closed. + */ + default CloseableIteration getStatements(StatementOrder statementOrder, Resource subj, + IRI pred, Value obj, + boolean includeInferred, Resource... contexts) throws SailException { + throw new SailException("Statement ordering is not supported by this SailConnection"); + } + /** * Determines if the store contains any statements from the specified contexts that have a specific subject, * predicate and/or object. All three parameters may be null to indicate wildcards. The includeInferred @@ -441,4 +467,8 @@ default Explanation explain(Explanation.Level level, TupleExpr tupleExpr, Datase throw new UnsupportedOperationException(); } + default Set getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) { + return Set.of(); + } + } diff --git a/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java b/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java index b0067d0292e..3e45f5bbe1f 100644 --- a/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java +++ b/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java @@ -29,6 +29,7 @@ import org.eclipse.rdf4j.common.concurrent.locks.diagnostics.ConcurrentCleaner; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.EmptyIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.common.transaction.IsolationLevel; import org.eclipse.rdf4j.common.transaction.IsolationLevels; import org.eclipse.rdf4j.model.BNode; @@ -394,6 +395,29 @@ public final CloseableIteration getStatements(Resource subj } } + @Override + public final CloseableIteration getStatements(StatementOrder order, Resource subj, IRI pred, + Value obj, boolean includeInferred, Resource... contexts) throws SailException { + flushPendingUpdates(); + + blockClose.increment(); + try { + verifyIsOpen(); + CloseableIteration iteration = null; + try { + iteration = getStatementsInternal(order, subj, pred, obj, includeInferred, contexts); + return registerIteration(iteration); + } catch (Throwable t) { + if (iteration != null) { + iteration.close(); + } + throw t; + } + } finally { + unblockClose.increment(); + } + } + @Override public final boolean hasStatement(Resource subj, IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException { @@ -860,6 +884,11 @@ protected abstract CloseableIteration getContextIDsInternal( protected abstract CloseableIteration getStatementsInternal(Resource subj, IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException; + protected CloseableIteration getStatementsInternal(StatementOrder order, Resource subj, + IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException { + throw new SailException("StatementOrder not supported"); + } + protected abstract long sizeInternal(Resource... contexts) throws SailException; protected abstract void startTransactionInternal() throws SailException; diff --git a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailDataset.java b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailDataset.java index 45e12e39a4f..877b300d010 100644 --- a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailDataset.java +++ b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailDataset.java @@ -10,7 +10,11 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.base; +import java.util.Comparator; +import java.util.Set; + import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.common.transaction.IsolationLevels; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Namespace; @@ -79,6 +83,26 @@ public interface SailDataset extends SailClosable { CloseableIteration getStatements(Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException; + /** + * Gets all statements that have a specific subject, predicate and/or object. All three parameters may be null to + * indicate wildcards. Optionally a (set of) context(s) may be specified in which case the result will be restricted + * to statements matching one or more of the specified contexts. + * + * @param statementOrder The order that the statements should be returned in. + * @param subj A Resource specifying the subject, or null for a wildcard. + * @param pred A IRI specifying the predicate, or null for a wildcard. + * @param obj A Value specifying the object, or null for a wildcard. + * @param contexts The context(s) to get the statements from. Note that this parameter is a vararg and as such + * is optional. If no contexts are supplied the method operates on all contexts. + * @return An iterator over the relevant statements. + * @throws SailException If the triple source failed to get the statements. + */ + default CloseableIteration getStatements(StatementOrder statementOrder, Resource subj, + IRI pred, Value obj, + Resource... contexts) throws SailException { + throw new SailException("Statement ordering not supported by this store"); + } + /** * Gets all RDF-star triples that have a specific subject, predicate and/or object. All three parameters may be null * to indicate wildcards. @@ -94,4 +118,12 @@ default CloseableIteration getTriples(Resource subj, IRI pred, throw new SailException("RDF-star triple retrieval not supported by this store"); } + default Set getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) { + return Set.of(); + } + + default Comparator getComparator(StatementOrder statementOrder) { + throw new SailException("Statement ordering not supported by this store"); + } + } diff --git a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java index f4048b26aa9..e69e7422a69 100644 --- a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java +++ b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java @@ -17,6 +17,7 @@ import java.util.stream.Stream; import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.common.transaction.IsolationLevel; import org.eclipse.rdf4j.common.transaction.IsolationLevels; import org.eclipse.rdf4j.common.transaction.QueryEvaluationMode; @@ -404,6 +405,16 @@ protected CloseableIteration getStatementsInternal(Resource return SailClosingIteration.makeClosable(snapshot.getStatements(subj, pred, obj, contexts), snapshot, branch); } + @Override + protected CloseableIteration getStatementsInternal(StatementOrder order, Resource subj, + IRI pred, + Value obj, boolean includeInferred, Resource... contexts) throws SailException { + SailSource branch = branch(IncludeInferred.fromBoolean(includeInferred)); + SailDataset snapshot = branch.dataset(getIsolationLevel()); + return SailClosingIteration.makeClosable(snapshot.getStatements(order, subj, pred, obj, contexts), snapshot, + branch); + } + @Override protected long sizeInternal(Resource... contexts) throws SailException { try (Stream stream = getStatementsInternal(null, null, null, false, contexts).stream()) { diff --git a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/UnionSailDataset.java b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/UnionSailDataset.java index 060ae133e5f..c0be6168e67 100644 --- a/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/UnionSailDataset.java +++ b/core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/UnionSailDataset.java @@ -11,8 +11,13 @@ package org.eclipse.rdf4j.sail.base; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Set; + import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.DualUnionIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Namespace; import org.eclipse.rdf4j.model.Resource; @@ -171,4 +176,53 @@ private CloseableIteration union( return DualUnionIteration.getWildcardInstance(iteration1, iteration2); } + private CloseableIteration union(Comparator cmp, + CloseableIteration iteration1, + CloseableIteration iteration2) { + return DualUnionIteration.getWildcardInstance(cmp, iteration1, iteration2); + } + + @Override + public CloseableIteration getStatements(StatementOrder statementOrder, Resource subj, IRI pred, + Value obj, Resource... contexts) throws SailException { + + CloseableIteration iteration1 = null; + CloseableIteration iteration2 = null; + try { + iteration1 = dataset1.getStatements(statementOrder, subj, pred, obj, contexts); + iteration2 = dataset2.getStatements(statementOrder, subj, pred, obj, contexts); + return union(dataset1.getComparator(statementOrder), iteration1, iteration2); + } catch (Throwable t) { + try { + if (iteration1 != null) { + iteration1.close(); + } + } finally { + if (iteration2 != null) { + iteration2.close(); + } + } + throw t; + } + + } + + @Override + public Set getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) { + Set availableOrderings1 = dataset1.getAvailableOrderings(subj, pred, obj, contexts); + if (availableOrderings1.isEmpty()) { + return Set.of(); + } + Set availableOrderings2 = dataset2.getAvailableOrderings(subj, pred, obj, contexts); + if (availableOrderings2.isEmpty()) { + return Set.of(); + } + if (availableOrderings1.equals(availableOrderings2)) { + return availableOrderings1; + } + + EnumSet commonStatementOrderings = EnumSet.copyOf(availableOrderings1); + commonStatementOrderings.retainAll(availableOrderings2); + return commonStatementOrderings; + } } diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/DataStructureInterface.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/DataStructureInterface.java index c78c7fbfa80..01101ec0b91 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/DataStructureInterface.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/DataStructureInterface.java @@ -11,12 +11,17 @@ package org.eclipse.rdf4j.sail.extensiblestore; import java.util.Collection; +import java.util.Comparator; +import java.util.Set; import org.eclipse.rdf4j.common.annotation.Experimental; import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.sail.SailException; import org.eclipse.rdf4j.sail.extensiblestore.valuefactory.ExtensibleStatement; /** @@ -48,9 +53,15 @@ CloseableIteration getStatements( IRI predicate, Value object, boolean inferred, - Resource... context); + Resource... contexts); // flush this DataStructure to make added and removed data visible to read operations + + default CloseableIteration getStatements(StatementOrder statementOrder, + Resource subject, IRI predicate, Value object, boolean inferred, Resource... contexts) { + throw new SailException("StatementOrder not supported"); + } + void flushForReading(); void init(); @@ -89,4 +100,14 @@ default long getEstimatedSize() { long explicit = getStatements(null, null, null, false).stream().count(); return inferred + explicit; } + + default Set getAvailableOrderings(Resource subj, IRI pred, Value obj, boolean inferred, + Resource... contexts) { + return Set.of(); + } + + default Comparator getComparator(StatementOrder statementOrder) { + throw new SailException("StatementOrder not supported"); + } + } diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/EagerReadCache.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/EagerReadCache.java index 29da5bdd610..4c05e0dfaf1 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/EagerReadCache.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/EagerReadCache.java @@ -10,10 +10,12 @@ ******************************************************************************/ package org.eclipse.rdf4j.sail.extensiblestore; +import java.util.Comparator; import java.util.Iterator; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.LookAheadIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Model; import org.eclipse.rdf4j.model.Resource; @@ -247,4 +249,9 @@ public long getEstimatedSize() { } return delegate.getEstimatedSize(); } + + @Override + public Comparator getComparator(StatementOrder statementOrder) { + throw new SailException("StatementOrder not supported"); + } } diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ExtensibleSailSource.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ExtensibleSailSource.java index a49e9aced28..defa5f05473 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ExtensibleSailSource.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ExtensibleSailSource.java @@ -10,12 +10,14 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.extensiblestore; +import java.util.Comparator; import java.util.HashSet; import java.util.Set; import org.eclipse.rdf4j.common.annotation.Experimental; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.CloseableIteratorIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.common.transaction.IsolationLevel; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Namespace; @@ -27,6 +29,7 @@ import org.eclipse.rdf4j.sail.base.SailDataset; import org.eclipse.rdf4j.sail.base.SailSink; import org.eclipse.rdf4j.sail.base.SailSource; +import org.eclipse.rdf4j.sail.extensiblestore.valuefactory.ExtensibleStatement; import org.eclipse.rdf4j.sail.extensiblestore.valuefactory.ExtensibleStatementHelper; /** @@ -209,6 +212,20 @@ public CloseableIteration getStatements(Resource subj, IRI return dataStructure.getStatements(subj, pred, obj, inferred, contexts); } + @Override + public CloseableIteration getStatements(StatementOrder order, Resource subj, IRI pred, + Value obj, Resource... contexts) throws SailException { + return dataStructure.getStatements(order, subj, pred, obj, inferred, contexts); + } + + @Override + public Set getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) { + return dataStructure.getAvailableOrderings(subj, pred, obj, inferred, contexts); + } + + public Comparator getComparator(StatementOrder statementOrder) { + return dataStructure.getComparator(statementOrder); + } }; } diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/LazyReadCache.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/LazyReadCache.java index c0cf2ffd99f..508a7998cbb 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/LazyReadCache.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/LazyReadCache.java @@ -11,11 +11,13 @@ package org.eclipse.rdf4j.sail.extensiblestore; import java.util.ArrayList; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.iteration.LookAheadIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.Value; @@ -194,6 +196,11 @@ synchronized public void submitToCache(Long localCacheTicket, PartialStatement p } + @Override + public Comparator getComparator(StatementOrder statementOrder) { + return delegate.getComparator(statementOrder); + } + @Override public long getEstimatedSize() { return delegate.getEstimatedSize(); diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java index f67b8046414..c6d5597fd83 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/ReadCommittedWrapper.java @@ -10,6 +10,7 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.extensiblestore; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -22,6 +23,7 @@ import org.eclipse.rdf4j.common.iteration.EmptyIteration; import org.eclipse.rdf4j.common.iteration.LookAheadIteration; import org.eclipse.rdf4j.common.iteration.SingletonIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.Statement; @@ -201,6 +203,11 @@ public void flushForCommit() { public void flushForReading() { } + @Override + public Comparator getComparator(StatementOrder statementOrder) { + return dataStructure.getComparator(statementOrder); + } + @Override public void init() { dataStructure.init(); diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/SortedIteration.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/SortedIteration.java new file mode 100644 index 00000000000..102199a3584 --- /dev/null +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/SortedIteration.java @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright (c) 2019 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ + +package org.eclipse.rdf4j.sail.extensiblestore; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +import org.eclipse.rdf4j.common.annotation.Experimental; +import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.iteration.Iterations; +import org.eclipse.rdf4j.common.iteration.LookAheadIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; +import org.eclipse.rdf4j.sail.extensiblestore.valuefactory.ExtensibleStatement; + +/** + * A wrapper for an Iteration that filters the statements against a pattern similar to getStatements(Resource subject, + * IRI predicate, Value object, Resource... context). + */ +@Experimental +public class SortedIteration extends LookAheadIteration { + + private final CloseableIteration wrappedIteration; + private final StatementOrder statementOrder; + private final Comparator comparator; + private boolean initialized; + private Iterator orderedIterator; + + public SortedIteration(CloseableIteration wrappedIteration, StatementOrder statementOrder) { + this.wrappedIteration = wrappedIteration; + this.statementOrder = statementOrder; + + if (statementOrder.equals(StatementOrder.S)) { + comparator = Comparator.comparing(o -> o.getSubject().toString()); + } else if (statementOrder.equals(StatementOrder.P)) { + comparator = Comparator.comparing(o -> o.getPredicate().toString()); + } else if (statementOrder.equals(StatementOrder.O)) { + comparator = Comparator.comparing(o -> o.getObject().toString()); + } else if (statementOrder.equals(StatementOrder.C)) { + comparator = Comparator.comparing(o -> o.getContext().toString()); + } else { + throw new IllegalArgumentException("Unknown StatementOrder: " + statementOrder); + } + } + + private void lazyInit() { + if (initialized) { + return; + } + + initialized = true; + + try (wrappedIteration) { + List list = Iterations.asList(wrappedIteration); + list.sort(comparator); + orderedIterator = list.iterator(); + } + + } + + @Override + protected E getNextElement() { + lazyInit(); + if (orderedIterator == null) { + throw new NoSuchElementException("Iteration has been closed"); + } + if (orderedIterator.hasNext()) { + return orderedIterator.next(); + } + return null; + + } + + @Override + protected void handleClose() { + wrappedIteration.close(); + orderedIterator = null; + } + +} diff --git a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/evaluationstatistics/EvaluationStatisticsWrapper.java b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/evaluationstatistics/EvaluationStatisticsWrapper.java index 117770c998a..1011ff326d3 100644 --- a/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/evaluationstatistics/EvaluationStatisticsWrapper.java +++ b/core/sail/extensible-store/src/main/java/org/eclipse/rdf4j/sail/extensiblestore/evaluationstatistics/EvaluationStatisticsWrapper.java @@ -11,9 +11,12 @@ package org.eclipse.rdf4j.sail.extensiblestore.evaluationstatistics; import java.util.Collection; +import java.util.Comparator; +import java.util.Set; import org.eclipse.rdf4j.common.annotation.Experimental; import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.Value; @@ -55,6 +58,12 @@ public CloseableIteration getStatements(Resource return delegate.getStatements(subject, predicate, object, inferred, context); } + @Override + public CloseableIteration getStatements(StatementOrder statementOrder, + Resource subject, IRI predicate, Value object, boolean inferred, Resource... contexts) { + return delegate.getStatements(statementOrder, subject, predicate, object, inferred, contexts); + } + @Override public void flushForReading() { delegate.flushForReading(); @@ -98,6 +107,17 @@ public long getEstimatedSize() { return delegate.getEstimatedSize(); } + @Override + public Set getAvailableOrderings(Resource subj, IRI pred, Value obj, boolean inferred, + Resource... contexts) { + return delegate.getAvailableOrderings(subj, pred, obj, inferred, contexts); + } + + @Override + public Comparator getComparator(StatementOrder statementOrder) { + return delegate.getComparator(statementOrder); + } + public void setEvaluationStatistics(DynamicStatistics dynamicStatistics) { this.dynamicStatistics = dynamicStatistics; } diff --git a/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreConnectionOrderedImplForTests.java b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreConnectionOrderedImplForTests.java new file mode 100644 index 00000000000..2bacad0c94a --- /dev/null +++ b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreConnectionOrderedImplForTests.java @@ -0,0 +1,20 @@ +/******************************************************************************* + * Copyright (c) 2019 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.extensiblestore.ordered; + +import org.eclipse.rdf4j.sail.extensiblestore.ExtensibleStoreConnection; + +public class ExtensibleStoreConnectionOrderedImplForTests + extends ExtensibleStoreConnection { + protected ExtensibleStoreConnectionOrderedImplForTests(ExtensibleStoreOrderedImplForTests sail) { + super(sail); + } +} diff --git a/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreOrderedImplForTests.java b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreOrderedImplForTests.java new file mode 100644 index 00000000000..2a9b0773e8a --- /dev/null +++ b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/ExtensibleStoreOrderedImplForTests.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2019 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.extensiblestore.ordered; + +import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics; +import org.eclipse.rdf4j.sail.NotifyingSailConnection; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.sail.extensiblestore.ExtensibleStore; +import org.eclipse.rdf4j.sail.extensiblestore.SimpleMemoryNamespaceStore; + +public class ExtensibleStoreOrderedImplForTests + extends ExtensibleStore { + + public ExtensibleStoreOrderedImplForTests() { + super(Cache.NONE); + } + + public ExtensibleStoreOrderedImplForTests(Cache cache) { + super(cache); + } + + @Override + protected synchronized void initializeInternal() throws SailException { + namespaceStore = new SimpleMemoryNamespaceStore(); + dataStructure = new OrderedDataStructure(); + super.initializeInternal(); + } + + @Override + protected NotifyingSailConnection getConnectionInternal() throws SailException { + return new ExtensibleStoreConnectionOrderedImplForTests(this); + } + + @Override + public boolean isWritable() throws SailException { + return true; + } + + public EvaluationStatistics getEvalStats() { + return sailStore.getEvaluationStatistics(); + } +} diff --git a/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedDataStructure.java b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedDataStructure.java new file mode 100644 index 00000000000..01196efd2dc --- /dev/null +++ b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedDataStructure.java @@ -0,0 +1,110 @@ +/******************************************************************************* + * Copyright (c) 2019 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.extensiblestore.ordered; + +import java.util.Collections; +import java.util.Comparator; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.iteration.CloseableIteratorIteration; +import org.eclipse.rdf4j.common.iteration.EmptyIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.sail.extensiblestore.DataStructureInterface; +import org.eclipse.rdf4j.sail.extensiblestore.FilteringIteration; +import org.eclipse.rdf4j.sail.extensiblestore.SortedIteration; +import org.eclipse.rdf4j.sail.extensiblestore.valuefactory.ExtensibleStatement; + +public class OrderedDataStructure implements DataStructureInterface { + + private static final EmptyIteration EMPTY_ITERATION = new EmptyIteration<>(); + + Set statements = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + @Override + synchronized public void addStatement(ExtensibleStatement statement) { + statements.add(statement); + } + + @Override + synchronized public void removeStatement(ExtensibleStatement statement) { + statements.remove(statement); + + } + + @Override + synchronized public CloseableIteration getStatements(Resource subject, + IRI predicate, + Value object, boolean inferred, Resource... context) { + return new FilteringIteration<>( + new CloseableIteratorIteration<>(statements.iterator()), subject, predicate, object, inferred, context); + } + + @Override + public CloseableIteration getStatements(StatementOrder statementOrder, + Resource subject, IRI predicate, Value object, boolean inferred, Resource... contexts) { + if (statements.isEmpty()) { + return EMPTY_ITERATION; + } + if (inferred) { + boolean containsInferred = statements.stream().anyMatch(ExtensibleStatement::isInferred); + if (!containsInferred) + return EMPTY_ITERATION; + } + return new SortedIteration<>(new FilteringIteration<>(new CloseableIteratorIteration<>(statements.iterator()), + subject, predicate, object, inferred, contexts), statementOrder); + } + + @Override + public void flushForReading() { + + } + + @Override + public void init() { + + } + + @Override + public void flushForCommit() { + + } + + @Override + public long getEstimatedSize() { + return statements.size(); + } + + @Override + public Set getAvailableOrderings(Resource subj, IRI pred, Value obj, boolean inferred, + Resource... contexts) { + return Set.of(StatementOrder.S, StatementOrder.P, StatementOrder.O, StatementOrder.C); + } + + @Override + public Comparator getComparator(StatementOrder statementOrder) { + if (statementOrder.equals(StatementOrder.S)) { + return Comparator.comparing(o -> o.getSubject().toString()); + } else if (statementOrder.equals(StatementOrder.P)) { + return Comparator.comparing(o -> o.getPredicate().toString()); + } else if (statementOrder.equals(StatementOrder.O)) { + return Comparator.comparing(o -> o.getObject().toString()); + } else if (statementOrder.equals(StatementOrder.C)) { + return Comparator.comparing(o -> o.getContext().toString()); + } else { + throw new IllegalArgumentException("Unknown StatementOrder: " + statementOrder); + } + } +} diff --git a/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedTest.java b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedTest.java new file mode 100644 index 00000000000..ce3b90011d4 --- /dev/null +++ b/core/sail/extensible-store/src/test/java/org/eclipse/rdf4j/sail/extensiblestore/ordered/OrderedTest.java @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright (c) 2023 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +package org.eclipse.rdf4j.sail.extensiblestore.ordered; + +import java.util.List; +import java.util.stream.Collectors; + +import org.assertj.core.api.Assertions; +import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.ordering.StatementOrder; +import org.eclipse.rdf4j.common.transaction.IsolationLevels; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.util.Values; +import org.eclipse.rdf4j.model.vocabulary.RDFS; +import org.eclipse.rdf4j.sail.NotifyingSailConnection; +import org.junit.jupiter.api.Test; + +public class OrderedTest { + + public static final String NAMESPACE = "http://example.com/"; + + @Test + public void test() { + ExtensibleStoreOrderedImplForTests store = new ExtensibleStoreOrderedImplForTests(); + + try (NotifyingSailConnection connection = store.getConnection()) { + connection.begin(); + connection.addStatement(Values.iri(NAMESPACE, "d"), RDFS.LABEL, Values.literal("b")); + connection.addStatement(Values.iri(NAMESPACE, "e"), RDFS.LABEL, Values.literal("a")); + connection.addStatement(Values.iri(NAMESPACE, "a"), RDFS.LABEL, Values.literal("e")); + connection.addStatement(Values.iri(NAMESPACE, "c"), RDFS.LABEL, Values.literal("c")); + connection.addStatement(Values.iri(NAMESPACE, "b"), RDFS.LABEL, Values.literal("d")); + + connection.commit(); + + connection.begin(IsolationLevels.NONE); + try (CloseableIteration statements = connection.getStatements(StatementOrder.S, null, + null, null, true)) { + List collect = statements.stream().collect(Collectors.toList()); + + List subjects = collect + .stream() + .map(Statement::getSubject) + .map(i -> (IRI) i) + .map(IRI::getLocalName) + .collect(Collectors.toList()); + + Assertions.assertThat(subjects).isEqualTo(List.of("a", "b", "c", "d", "e")); + } + + connection.commit(); + } + } + + @Test + public void testReadCommitted() { + ExtensibleStoreOrderedImplForTests store = new ExtensibleStoreOrderedImplForTests(); + + try (NotifyingSailConnection connection = store.getConnection()) { + connection.begin(); + connection.addStatement(Values.iri(NAMESPACE, "a"), RDFS.LABEL, Values.literal("e")); + connection.addStatement(Values.iri(NAMESPACE, "b"), RDFS.LABEL, Values.literal("d")); + connection.addStatement(Values.iri(NAMESPACE, "c"), RDFS.LABEL, Values.literal("c")); + connection.addStatement(Values.iri(NAMESPACE, "d"), RDFS.LABEL, Values.literal("b")); + connection.addStatement(Values.iri(NAMESPACE, "e"), RDFS.LABEL, Values.literal("a")); + connection.commit(); + + connection.begin(IsolationLevels.READ_COMMITTED); + try (CloseableIteration statements = connection.getStatements(StatementOrder.S, null, + null, null, true)) { + List collect = statements.stream().collect(Collectors.toList()); + + List subjects = collect + .stream() + .map(Statement::getSubject) + .map(i -> (IRI) i) + .map(IRI::getLocalName) + .collect(Collectors.toList()); + + Assertions.assertThat(subjects).isEqualTo(List.of("a", "b", "c", "d", "e")); + } + + connection.commit(); + } + } + +}