GH-4819 initial support for specifying statement order from the SailC…

…onnection through to the ExtensibleStore datastructure, without support for inferred statements, ReadCommitted or the caching layers

core/common/iterator/pom.xml

@@ -15,6 +15,11 @@
 			<artifactId>rdf4j-model-api</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>rdf4j-common-ordering</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>

core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DualUnionIteration.java

@@ -11,8 +11,11 @@
 
 package org.eclipse.rdf4j.common.iteration;
 
+import java.util.Comparator;
 import java.util.NoSuchElementException;
 
+import org.eclipse.rdf4j.common.ordering.StatementOrder;
+
 /**
  * Provides a bag union of the two provided iterations.
  */
@@ -32,6 +35,12 @@ private DualUnionIteration(CloseableIteration<? extends E> iteration1,
 		this.iteration2 = iteration2;
 	}
 
+	public <E> DualUnionIteration(Comparator<E> cmp, CloseableIteration<? extends E> leftIteration,
+			CloseableIteration<? extends E> rightIteration) {
+		throw new UnsupportedOperationException("Not implemented yet");
+
+	}
+
 	public static <E, X extends Exception> CloseableIteration<? extends E> getWildcardInstance(
 			CloseableIteration<? extends E> leftIteration, CloseableIteration<? extends E> rightIteration) {
 
@@ -44,6 +53,18 @@ public static <E, X extends Exception> CloseableIteration<? extends E> getWildca
 		}
 	}
 
+	public static <E, X extends Exception> CloseableIteration<? extends E> getWildcardInstance(Comparator<E> cmp,
+			CloseableIteration<? extends E> leftIteration, CloseableIteration<? extends E> rightIteration) {
+
+		if (rightIteration instanceof EmptyIteration) {
+			return leftIteration;
+		} else if (leftIteration instanceof EmptyIteration) {
+			return rightIteration;
+		} else {
+			return new DualUnionIteration<>(cmp, leftIteration, rightIteration);
+		}
+	}
+
 	public static <E, X extends Exception> CloseableIteration<E> getInstance(CloseableIteration<E> leftIteration,
 			CloseableIteration<E> rightIteration) {
 

core/common/ordering/pom.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.eclipse.rdf4j</groupId>
+		<artifactId>rdf4j-common</artifactId>
+		<version>5.0.0-SNAPSHOT</version>
+	</parent>
+	<artifactId>rdf4j-common-ordering</artifactId>
+	<name>RDF4J: common ordering</name>
+	<description>RDF4J common ordering classes</description>
+	<dependencies/>
+	<build>
+		<plugins>
+			<plugin>
+				<artifactId>maven-assembly-plugin</artifactId>
+			</plugin>
+		</plugins>
+	</build>
+</project>

core/common/ordering/src/main/java/org/eclipse/rdf4j/common/ordering/StatementOrder.java

@@ -0,0 +1,21 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Eclipse RDF4J contributors.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ ******************************************************************************/
+
+package org.eclipse.rdf4j.common.ordering;
+
+public enum StatementOrder {
+
+	S,
+	P,
+	O,
+	C;
+
+}

core/common/pom.xml

@@ -15,6 +15,7 @@
 		<module>exception</module>
 		<module>io</module>
 		<module>iterator</module>
+		<module>ordering</module>
 		<module>text</module>
 		<module>transaction</module>
 		<module>xml</module>

core/sail/api/pom.xml

@@ -30,6 +30,11 @@
 			<artifactId>rdf4j-common-transaction</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>rdf4j-common-ordering</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 		<dependency>
 			<groupId>${project.groupId}</groupId>
 			<artifactId>rdf4j-collection-factory-api</artifactId>

core/sail/api/src/main/java/org/eclipse/rdf4j/sail/SailConnection.java

@@ -11,9 +11,11 @@
 package org.eclipse.rdf4j.sail;
 
 import java.util.Optional;
+import java.util.Set;
 
 import org.eclipse.rdf4j.common.annotation.Experimental;
 import org.eclipse.rdf4j.common.iteration.CloseableIteration;
+import org.eclipse.rdf4j.common.ordering.StatementOrder;
 import org.eclipse.rdf4j.common.transaction.IsolationLevel;
 import org.eclipse.rdf4j.common.transaction.TransactionSetting;
 import org.eclipse.rdf4j.model.IRI;
@@ -118,6 +120,30 @@ CloseableIteration<? extends BindingSet> evaluate(TupleExpr tupleExpr,
 	CloseableIteration<? extends Statement> getStatements(Resource subj, IRI pred, Value obj,
 			boolean includeInferred, Resource... contexts) throws SailException;
 
+	/**
+	 * Gets all statements from the specified contexts that have a specific subject, predicate and/or object. All three
+	 * parameters may be null to indicate wildcards. The <var>includeInferred</var> parameter can be used to control
+	 * which statements are fetched: all statements or only the statements that have been added explicitly.
+	 *
+	 * @param statementOrder  The order that the statements should be returned in.
+	 * @param subj            A Resource specifying the subject, or <var>null</var> for a wildcard.
+	 * @param pred            A URI specifying the predicate, or <var>null</var> for a wildcard.
+	 * @param obj             A Value specifying the object, or <var>null</var> for a wildcard.
+	 * @param includeInferred if false, no inferred statements are returned; if true, inferred statements are returned
+	 *                        if available
+	 * @param contexts        The context(s) to get the data from. Note that this parameter is a vararg and as such is
+	 *                        optional. If no contexts are specified the method operates on the entire repository. A
+	 *                        <var>null</var> value can be used to match context-less statements.
+	 * @return The statements matching the specified pattern.
+	 * @throws SailException         If the Sail object encountered an error or unexpected situation internally.
+	 * @throws IllegalStateException If the connection has been closed.
+	 */
+	default CloseableIteration<? extends Statement> getStatements(StatementOrder statementOrder, Resource subj,
+			IRI pred, Value obj,
+			boolean includeInferred, Resource... contexts) throws SailException {
+		throw new SailException("Statement ordering is not supported by this SailConnection");
+	}
+
 	/**
 	 * Determines if the store contains any statements from the specified contexts that have a specific subject,
 	 * predicate and/or object. All three parameters may be null to indicate wildcards. The <var>includeInferred</var>
@@ -441,4 +467,8 @@ default Explanation explain(Explanation.Level level, TupleExpr tupleExpr, Datase
 		throw new UnsupportedOperationException();
 	}
 
+	default Set<StatementOrder> getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) {
+		return Set.of();
+	}
+
 }

core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java

@@ -29,6 +29,7 @@
 import org.eclipse.rdf4j.common.concurrent.locks.diagnostics.ConcurrentCleaner;
 import org.eclipse.rdf4j.common.iteration.CloseableIteration;
 import org.eclipse.rdf4j.common.iteration.EmptyIteration;
+import org.eclipse.rdf4j.common.ordering.StatementOrder;
 import org.eclipse.rdf4j.common.transaction.IsolationLevel;
 import org.eclipse.rdf4j.common.transaction.IsolationLevels;
 import org.eclipse.rdf4j.model.BNode;
@@ -394,6 +395,29 @@ public final CloseableIteration<? extends Statement> getStatements(Resource subj
 		}
 	}
 
+	@Override
+	public final CloseableIteration<? extends Statement> getStatements(StatementOrder order, Resource subj, IRI pred,
+			Value obj, boolean includeInferred, Resource... contexts) throws SailException {
+		flushPendingUpdates();
+
+		blockClose.increment();
+		try {
+			verifyIsOpen();
+			CloseableIteration<? extends Statement> iteration = null;
+			try {
+				iteration = getStatementsInternal(order, subj, pred, obj, includeInferred, contexts);
+				return registerIteration(iteration);
+			} catch (Throwable t) {
+				if (iteration != null) {
+					iteration.close();
+				}
+				throw t;
+			}
+		} finally {
+			unblockClose.increment();
+		}
+	}
+
 	@Override
 	public final boolean hasStatement(Resource subj, IRI pred, Value obj, boolean includeInferred, Resource... contexts)
 			throws SailException {
@@ -860,6 +884,11 @@ protected abstract CloseableIteration<? extends Resource> getContextIDsInternal(
 	protected abstract CloseableIteration<? extends Statement> getStatementsInternal(Resource subj,
 			IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException;
 
+	protected CloseableIteration<? extends Statement> getStatementsInternal(StatementOrder order, Resource subj,
+			IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException {
+		throw new SailException("StatementOrder not supported");
+	}
+
 	protected abstract long sizeInternal(Resource... contexts) throws SailException;
 
 	protected abstract void startTransactionInternal() throws SailException;

core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailDataset.java

@@ -10,7 +10,11 @@
  *******************************************************************************/
 package org.eclipse.rdf4j.sail.base;
 
+import java.util.Comparator;
+import java.util.Set;
+
 import org.eclipse.rdf4j.common.iteration.CloseableIteration;
+import org.eclipse.rdf4j.common.ordering.StatementOrder;
 import org.eclipse.rdf4j.common.transaction.IsolationLevels;
 import org.eclipse.rdf4j.model.IRI;
 import org.eclipse.rdf4j.model.Namespace;
@@ -79,6 +83,26 @@ public interface SailDataset extends SailClosable {
 	CloseableIteration<? extends Statement> getStatements(Resource subj, IRI pred, Value obj,
 			Resource... contexts) throws SailException;
 
+	/**
+	 * Gets all statements that have a specific subject, predicate and/or object. All three parameters may be null to
+	 * indicate wildcards. Optionally a (set of) context(s) may be specified in which case the result will be restricted
+	 * to statements matching one or more of the specified contexts.
+	 *
+	 * @param statementOrder The order that the statements should be returned in.
+	 * @param subj           A Resource specifying the subject, or <var>null</var> for a wildcard.
+	 * @param pred           A IRI specifying the predicate, or <var>null</var> for a wildcard.
+	 * @param obj            A Value specifying the object, or <var>null</var> for a wildcard.
+	 * @param contexts       The context(s) to get the statements from. Note that this parameter is a vararg and as such
+	 *                       is optional. If no contexts are supplied the method operates on all contexts.
+	 * @return An iterator over the relevant statements.
+	 * @throws SailException If the triple source failed to get the statements.
+	 */
+	default CloseableIteration<? extends Statement> getStatements(StatementOrder statementOrder, Resource subj,
+			IRI pred, Value obj,
+			Resource... contexts) throws SailException {
+		throw new SailException("Statement ordering not supported by this store");
+	}
+
 	/**
 	 * Gets all RDF-star triples that have a specific subject, predicate and/or object. All three parameters may be null
 	 * to indicate wildcards.
@@ -94,4 +118,12 @@ default CloseableIteration<? extends Triple> getTriples(Resource subj, IRI pred,
 		throw new SailException("RDF-star triple retrieval not supported by this store");
 	}
 
+	default Set<StatementOrder> getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) {
+		return Set.of();
+	}
+
+	default Comparator<? extends Statement> getComparator(StatementOrder statementOrder) {
+		throw new SailException("Statement ordering not supported by this store");
+	}
+
 }

core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SailSourceConnection.java

@@ -17,6 +17,7 @@
 import java.util.stream.Stream;
 
 import org.eclipse.rdf4j.common.iteration.CloseableIteration;
+import org.eclipse.rdf4j.common.ordering.StatementOrder;
 import org.eclipse.rdf4j.common.transaction.IsolationLevel;
 import org.eclipse.rdf4j.common.transaction.IsolationLevels;
 import org.eclipse.rdf4j.common.transaction.QueryEvaluationMode;
@@ -404,6 +405,16 @@ protected CloseableIteration<? extends Statement> getStatementsInternal(Resource
 		return SailClosingIteration.makeClosable(snapshot.getStatements(subj, pred, obj, contexts), snapshot, branch);
 	}
 
+	@Override
+	protected CloseableIteration<? extends Statement> getStatementsInternal(StatementOrder order, Resource subj,
+			IRI pred,
+			Value obj, boolean includeInferred, Resource... contexts) throws SailException {
+		SailSource branch = branch(IncludeInferred.fromBoolean(includeInferred));
+		SailDataset snapshot = branch.dataset(getIsolationLevel());
+		return SailClosingIteration.makeClosable(snapshot.getStatements(order, subj, pred, obj, contexts), snapshot,
+				branch);
+	}
+
 	@Override
 	protected long sizeInternal(Resource... contexts) throws SailException {
 		try (Stream<? extends Statement> stream = getStatementsInternal(null, null, null, false, contexts).stream()) {

core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/UnionSailDataset.java

@@ -11,8 +11,13 @@
 
 package org.eclipse.rdf4j.sail.base;
 
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.Set;
+
 import org.eclipse.rdf4j.common.iteration.CloseableIteration;
 import org.eclipse.rdf4j.common.iteration.DualUnionIteration;
+import org.eclipse.rdf4j.common.ordering.StatementOrder;
 import org.eclipse.rdf4j.model.IRI;
 import org.eclipse.rdf4j.model.Namespace;
 import org.eclipse.rdf4j.model.Resource;
@@ -171,4 +176,53 @@ private <T> CloseableIteration<? extends T> union(
 		return DualUnionIteration.getWildcardInstance(iteration1, iteration2);
 	}
 
+	private <T> CloseableIteration<? extends T> union(Comparator cmp,
+			CloseableIteration<? extends T> iteration1,
+			CloseableIteration<? extends T> iteration2) {
+		return DualUnionIteration.getWildcardInstance(cmp, iteration1, iteration2);
+	}
+
+	@Override
+	public CloseableIteration<? extends Statement> getStatements(StatementOrder statementOrder, Resource subj, IRI pred,
+			Value obj, Resource... contexts) throws SailException {
+
+		CloseableIteration<? extends Statement> iteration1 = null;
+		CloseableIteration<? extends Statement> iteration2 = null;
+		try {
+			iteration1 = dataset1.getStatements(statementOrder, subj, pred, obj, contexts);
+			iteration2 = dataset2.getStatements(statementOrder, subj, pred, obj, contexts);
+			return union(dataset1.getComparator(statementOrder), iteration1, iteration2);
+		} catch (Throwable t) {
+			try {
+				if (iteration1 != null) {
+					iteration1.close();
+				}
+			} finally {
+				if (iteration2 != null) {
+					iteration2.close();
+				}
+			}
+			throw t;
+		}
+
+	}
+
+	@Override
+	public Set<StatementOrder> getAvailableOrderings(Resource subj, IRI pred, Value obj, Resource... contexts) {
+		Set<StatementOrder> availableOrderings1 = dataset1.getAvailableOrderings(subj, pred, obj, contexts);
+		if (availableOrderings1.isEmpty()) {
+			return Set.of();
+		}
+		Set<StatementOrder> availableOrderings2 = dataset2.getAvailableOrderings(subj, pred, obj, contexts);
+		if (availableOrderings2.isEmpty()) {
+			return Set.of();
+		}
+		if (availableOrderings1.equals(availableOrderings2)) {
+			return availableOrderings1;
+		}
+
+		EnumSet<StatementOrder> commonStatementOrderings = EnumSet.copyOf(availableOrderings1);
+		commonStatementOrderings.retainAll(availableOrderings2);
+		return commonStatementOrderings;
+	}
 }