diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/AccumulatingAsyncResultSet.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/AccumulatingAsyncResultSet.java index c290ab813..d094f2d7d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/AccumulatingAsyncResultSet.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/AccumulatingAsyncResultSet.java @@ -76,7 +76,7 @@ public Iterable currentPage() { @Override public int remaining() { - throw new NotImplementedException(); + return -1; } @Override diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/CqlPagingState.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/CqlPagingState.java index 7d47cb7a5..ab07e9723 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/CqlPagingState.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/CqlPagingState.java @@ -58,6 +58,18 @@ public Optional getPagingStateString() { return isEmpty ? Optional.empty() : Optional.of(pagingStateString); } + @Override + public String toString() { + return new StringBuilder("CqlPagingState{") + .append("isEmpty=") + .append(isEmpty) + .append(", pagingStateString='") + .append(pagingStateString) + .append('\'') + .append('}') + .toString(); + } + private static ByteBuffer decode(String pagingState) { return ByteBuffer.wrap(Base64.getDecoder().decode(pagingState)); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/ReadAttempt.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/ReadAttempt.java index ea3257dfa..8825b83a1 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/ReadAttempt.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/ReadAttempt.java @@ -201,6 +201,14 @@ static class ReadResult { this.resultSet = Objects.requireNonNull(resultSet, "resultSet must not be null"); this.currentPage = resultSet.currentPage(); this.pagingState = rowSorter.buildPagingState(resultSet); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "ReadResult() created resultSet.remaining: {}, resultSet.hasMorePages={}, (api) pagingState:{}", + resultSet.remaining(), + resultSet.hasMorePages(), + pagingState); + } } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ReadCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ReadCommandResolver.java index 1601ac567..1f5b22965 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ReadCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/ReadCommandResolver.java @@ -72,15 +72,21 @@ protected GenericOperation> bu var orderByWithWarnings = tableCqlSortClauseResolver.resolve(commandContext, command); attemptBuilder.addOrderBy(orderByWithWarnings); - // if the user did not provide a limit, we will use the default page size as read limit - int commandLimit = command.limit().orElseGet(operationsConfig::defaultPageSize); + // if the user did not provide a limit,we read all the possible rows. Paging is then handled + // by the driver pagination + int commandLimit = command.limit().orElseGet(() -> Integer.MAX_VALUE); int commandSkip = command.skip().orElse(0); // and then if we need to do in memory sorting var inMemorySort = new TableMemorySortClauseResolver<>( - operationsConfig, orderByWithWarnings.target(), commandSkip, commandLimit) + operationsConfig, + orderByWithWarnings.target(), + commandSkip, + // Math.min is used because the max documents the api return is + // `operationsConfig.defaultPageSize()` + Math.min(commandLimit, operationsConfig.defaultPageSize())) .resolve(commandContext, command); attemptBuilder.addSorter(inMemorySort); diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 6571cf441..10c822517 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -49,6 +49,8 @@ datastax-java-driver { } } basic.request.timeout = 20 seconds + // setting this at the global and the profile level + basic.request.page-size = 20 profiles { create { @@ -69,6 +71,9 @@ datastax-java-driver { table-read { basic.request.timeout = 10 seconds + // this will be the number of rows we return in a single page of results to the user. + // It may be overridded for in memory sorting, but this is the default. + basic.request.page-size = 20 consistency = LOCAL_QUORUM } table-write { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/FindNoPaginationTableIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/FindNoPaginationTableIntegrationTest.java new file mode 100644 index 000000000..c093dade9 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/FindNoPaginationTableIntegrationTest.java @@ -0,0 +1,41 @@ +package io.stargate.sgv2.jsonapi.api.v1.tables; + +import static io.stargate.sgv2.jsonapi.api.v1.util.DataApiCommandSenders.assertTableCommand; + +import io.quarkus.test.common.WithTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.stargate.sgv2.jsonapi.api.v1.util.scenarios.KeyValueTable10Scenario; +import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.*; + +@QuarkusIntegrationTest +@WithTestResource(value = DseTestResource.class, restrictToAnnotatedClass = false) +@TestClassOrder(ClassOrderer.OrderAnnotation.class) +public class FindNoPaginationTableIntegrationTest extends AbstractTableIntegrationTestBase { + + private static final String TABLE_NAME = "findPaginationTableIntegrationTest"; + private static final KeyValueTable10Scenario SCENARIO = + new KeyValueTable10Scenario(keyspaceName, TABLE_NAME); + + @BeforeAll + public final void createScenario() { + SCENARIO.create(); + } + + @AfterAll + public final void dropScenario() { + SCENARIO.drop(); + } + + @Test + public void findWithPageState() { + + assertTableCommand(keyspaceName, TABLE_NAME) + .templated() + .find(Map.of(), List.of()) + .wasSuccessful() + .doesNotHaveNextPageState(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/FindPaginationTableIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/FindPaginationTableIntegrationTest.java new file mode 100644 index 000000000..2ec3f466c --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/tables/FindPaginationTableIntegrationTest.java @@ -0,0 +1,41 @@ +package io.stargate.sgv2.jsonapi.api.v1.tables; + +import static io.stargate.sgv2.jsonapi.api.v1.util.DataApiCommandSenders.assertTableCommand; + +import io.quarkus.test.common.WithTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.stargate.sgv2.jsonapi.api.v1.util.scenarios.KeyValueTable100Scenario; +import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.*; + +@QuarkusIntegrationTest +@WithTestResource(value = DseTestResource.class, restrictToAnnotatedClass = false) +@TestClassOrder(ClassOrderer.OrderAnnotation.class) +public class FindPaginationTableIntegrationTest extends AbstractTableIntegrationTestBase { + + private static final String TABLE_NAME = "findPaginationTableIntegrationTest"; + private static final KeyValueTable100Scenario SCENARIO = + new KeyValueTable100Scenario(keyspaceName, TABLE_NAME); + + @BeforeAll + public final void createScenario() { + SCENARIO.create(); + } + + @AfterAll + public final void dropScenario() { + SCENARIO.drop(); + } + + @Test + public void findWithPageState() { + + assertTableCommand(keyspaceName, TABLE_NAME) + .templated() + .find(Map.of(), List.of()) + .wasSuccessful() + .hasNextPageState(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiResponseValidator.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiResponseValidator.java index cfd7c9b8f..37c0481e8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiResponseValidator.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/DataApiResponseValidator.java @@ -375,4 +375,12 @@ public DataApiResponseValidator doesNotHaveIndexes(String... indexes) { } return toReturn; } + + public DataApiResponseValidator hasNextPageState() { + return body("data.nextPageState", is(notNullValue())); + } + + public DataApiResponseValidator doesNotHaveNextPageState() { + return body("$", not(hasKey("data.nextPageState"))); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/scenarios/KeyValueTable100Scenario.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/scenarios/KeyValueTable100Scenario.java new file mode 100644 index 000000000..14ecca2ad --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/scenarios/KeyValueTable100Scenario.java @@ -0,0 +1,52 @@ +package io.stargate.sgv2.jsonapi.api.v1.util.scenarios; + +import com.datastax.oss.driver.api.core.CqlIdentifier; +import io.stargate.sgv2.jsonapi.fixtures.data.DefaultData; +import io.stargate.sgv2.jsonapi.service.schema.tables.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** basic key-value table with 100 rows */ +public class KeyValueTable100Scenario extends TestDataScenario { + + public static final ApiColumnDef VALUE_COL = + new ApiColumnDef(CqlIdentifier.fromCql("value_col"), ApiDataTypeDefs.TEXT); + + public KeyValueTable100Scenario(String keyspaceName, String tableName) { + super( + keyspaceName, + tableName, + ID_COL, + createClusteringDefs(), + createColumns(), + new DefaultData()); + } + + private static ApiColumnDefContainer createColumns() { + + var columns = new ApiColumnDefContainer(); + columns.put(ID_COL); + columns.put(VALUE_COL); + return columns; + } + + private static List createClusteringDefs() { + + return List.of(); + } + + @Override + protected void insertRows() { + + var rows = new ArrayList>(); + for (int i = 0; i < 100; i++) { + var row = new HashMap(); + row.put(fieldName(ID_COL), "row" + i); + row.put(fieldName(VALUE_COL), "value-" + i); + rows.add(row); + } + insertManyRows(rows); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/scenarios/KeyValueTable10Scenario.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/scenarios/KeyValueTable10Scenario.java new file mode 100644 index 000000000..5a7907c41 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/util/scenarios/KeyValueTable10Scenario.java @@ -0,0 +1,55 @@ +package io.stargate.sgv2.jsonapi.api.v1.util.scenarios; + +import com.datastax.oss.driver.api.core.CqlIdentifier; +import io.stargate.sgv2.jsonapi.fixtures.data.DefaultData; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiClusteringDef; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiColumnDef; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiColumnDefContainer; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiDataTypeDefs; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** basic key-value table with 10 rows */ +public class KeyValueTable10Scenario extends TestDataScenario { + + public static final ApiColumnDef VALUE_COL = + new ApiColumnDef(CqlIdentifier.fromCql("value_col"), ApiDataTypeDefs.TEXT); + + public KeyValueTable10Scenario(String keyspaceName, String tableName) { + super( + keyspaceName, + tableName, + ID_COL, + createClusteringDefs(), + createColumns(), + new DefaultData()); + } + + private static ApiColumnDefContainer createColumns() { + + var columns = new ApiColumnDefContainer(); + columns.put(ID_COL); + columns.put(VALUE_COL); + return columns; + } + + private static List createClusteringDefs() { + + return List.of(); + } + + @Override + protected void insertRows() { + + var rows = new ArrayList>(); + for (int i = 0; i < 10; i++) { + var row = new HashMap(); + row.put(fieldName(ID_COL), "row" + i); + row.put(fieldName(VALUE_COL), "value-" + i); + rows.add(row); + } + insertManyRows(rows); + } +}