diff --git a/docker-compose/README.md b/docker-compose/README.md index f2a421cfd7..fd69ada04b 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -1,4 +1,4 @@ -# Docker Compose scripts for JSONAPI with DSE-6.9 +# Docker Compose scripts for Data API with DSE-6.9 This directory provides two ways to start the Data API with DSE-6.9 or HCD using `docker compose`. diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandResult.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandResult.java index 154be44101..b546d192bc 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandResult.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandResult.java @@ -46,7 +46,7 @@ public record CommandResult( nullable = true) }) Map status, - @Schema(nullable = true) List errors) { + @JsonInclude(JsonInclude.Include.NON_EMPTY) @Schema(nullable = true) List errors) { /** * Constructor for only specifying the {@link MultiResponseData}. diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java index 67cc91b9e2..d56892a19b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/CommandStatus.java @@ -19,6 +19,14 @@ public enum CommandStatus { /** Status for reporting existing collections. */ @JsonProperty("collections") EXISTING_COLLECTIONS, + /** + * List of response entries, one for each document we tried to insert with {@code insertMany} + * command. Each entry has 2 mandatory fields: {@code _id} (document id), and {@code status} (one + * of {@code OK}, {@code ERROR} or {@code SKIP}; {@code ERROR} entries also have {@code errorsIdx} + * field that refers to position of the error in the root level {@code errors} List. + */ + @JsonProperty("documentResponses") + DOCUMENT_RESPONSES, /** The element has the list of inserted ids */ @JsonProperty("insertedIds") INSERTED_IDS, diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/InsertManyCommand.java b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/InsertManyCommand.java index 6928984e0a..46b832c42d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/InsertManyCommand.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/model/command/impl/InsertManyCommand.java @@ -38,5 +38,14 @@ public record Options( description = "When `true` the server will insert the documents in sequential order, ensuring each document is successfully inserted before starting the next. Additionally the command will \"fail fast\", failing the first document that fails to insert. When `false` the server is free to re-order the inserts and parallelize them for performance. In this mode more than one document may fail to be inserted (aka \"fail silently\" mode).", defaultValue = "false") - boolean ordered) {} + boolean ordered, + @Schema( + description = + "When `true`, response will contain an additional field: 'documentResponses'" + + " with is an array of Document Response Objects. Each Document Response Object" + + " contains the `_id` of the document and the `status` of the operation (one of" + + " `OK`, `ERROR` or `SKIPPED`). Additional `errorsIdx` field is present when the" + + " status is `ERROR` and contains the index of the error in the main `errors` array.", + defaultValue = "false") + boolean returnDocumentResponses) {} } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperation.java index d85c31ae45..e595f51679 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperation.java @@ -14,6 +14,7 @@ import io.stargate.sgv2.jsonapi.service.operation.model.ModifyOperation; import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId; import io.stargate.sgv2.jsonapi.service.shredding.model.WritableShreddedDocument; +import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.function.Supplier; @@ -29,16 +30,28 @@ public record InsertOperation( CommandContext commandContext, List documents, boolean ordered, - boolean offlineMode) + boolean offlineMode, + boolean returnDocumentResponses) implements ModifyOperation { + public record WritableDocAndPosition(int position, WritableShreddedDocument document) + implements Comparable { + @Override + public int compareTo(InsertOperation.WritableDocAndPosition o) { + // Order by position (only), ascending + return Integer.compare(position, o.position); + } + } public InsertOperation( - CommandContext commandContext, List documents, boolean ordered) { - this(commandContext, documents, ordered, false); + CommandContext commandContext, + List documents, + boolean ordered, + boolean returnDocumentResponses) { + this(commandContext, documents, ordered, false, returnDocumentResponses); } public InsertOperation(CommandContext commandContext, WritableShreddedDocument document) { - this(commandContext, List.of(document), false, false); + this(commandContext, List.of(document), false, false, false); } /** {@inheritDoc} */ @@ -57,22 +70,30 @@ public Uni> execute( .jsonProcessingMetricsReporter() .reportJsonWrittenDocsMetrics(commandContext().commandName(), documents.size()); } + final List docsWithPositions = new ArrayList<>(documents.size()); + int pos = 0; + for (WritableShreddedDocument doc : documents) { + docsWithPositions.add(new WritableDocAndPosition(pos++, doc)); + } if (ordered) { - return insertOrdered(dataApiRequestInfo, queryExecutor, vectorEnabled); + return insertOrdered(dataApiRequestInfo, queryExecutor, vectorEnabled, docsWithPositions); } else { - return insertUnordered(dataApiRequestInfo, queryExecutor, vectorEnabled); + return insertUnordered(dataApiRequestInfo, queryExecutor, vectorEnabled, docsWithPositions); } } // implementation for the ordered insert private Uni> insertOrdered( - DataApiRequestInfo dataApiRequestInfo, QueryExecutor queryExecutor, boolean vectorEnabled) { + DataApiRequestInfo dataApiRequestInfo, + QueryExecutor queryExecutor, + boolean vectorEnabled, + List docsWithPositions) { // build query once final String query = buildInsertQuery(vectorEnabled); return Multi.createFrom() - .iterable(documents) + .iterable(docsWithPositions) // concatenate to respect ordered .onItem() @@ -89,10 +110,10 @@ private Uni> insertOrdered( // if no failures reduce to the op page .collect() .in( - InsertOperationPage::new, + () -> new InsertOperationPage(docsWithPositions, returnDocumentResponses()), (agg, in) -> { Throwable failure = in.getItem2(); - agg.aggregate(in.getItem1().id(), failure); + agg.aggregate(in.getItem1(), failure); if (failure != null) { throw new FailFastInsertException(agg, failure); @@ -115,11 +136,14 @@ private Uni> insertOrdered( // implementation for the unordered insert private Uni> insertUnordered( - DataApiRequestInfo dataApiRequestInfo, QueryExecutor queryExecutor, boolean vectorEnabled) { + DataApiRequestInfo dataApiRequestInfo, + QueryExecutor queryExecutor, + boolean vectorEnabled, + List docsWithPositions) { // build query once String query = buildInsertQuery(vectorEnabled); return Multi.createFrom() - .iterable(documents) + .iterable(docsWithPositions) // merge to make it parallel .onItem() @@ -133,7 +157,9 @@ private Uni> insertUnordered( // then reduce here .collect() - .in(InsertOperationPage::new, (agg, in) -> agg.aggregate(in.getItem1().id(), in.getItem2())) + .in( + () -> new InsertOperationPage(docsWithPositions, returnDocumentResponses()), + (agg, in) -> agg.aggregate(in.getItem1(), in.getItem2())) // use object identity to resolve to Supplier .map(i -> i); @@ -144,10 +170,11 @@ private static Uni insertDocument( DataApiRequestInfo dataApiRequestInfo, QueryExecutor queryExecutor, String query, - WritableShreddedDocument doc, + WritableDocAndPosition docWithPosition, boolean vectorEnabled, boolean offlineMode) { // bind and execute + final WritableShreddedDocument doc = docWithPosition.document(); SimpleStatement boundStatement = bindInsertValues(query, doc, vectorEnabled, offlineMode); return queryExecutor .executeWrite(dataApiRequestInfo, boundStatement) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationPage.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationPage.java index 8a97b885fd..834fa4977d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationPage.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationPage.java @@ -1,12 +1,16 @@ package io.stargate.sgv2.jsonapi.service.operation.model.impl; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import io.smallrye.mutiny.tuples.Tuple2; import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; import io.stargate.sgv2.jsonapi.api.model.command.CommandStatus; -import io.stargate.sgv2.jsonapi.exception.JsonApiException; import io.stargate.sgv2.jsonapi.exception.mappers.ThrowableToErrorMapper; import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -15,59 +19,121 @@ * The internal to insert operation results, keeping ids of successfully and not-successfully * inserted documents. * - *

Can serve as an aggregator, using the {@link #aggregate(DocumentId, Throwable)} function. + *

Can serve as an aggregator, using the {@link #aggregate} function. * - * @param insertedIds Documents IDs that we successfully inserted. - * @param failedIds Document IDs that failed to be inserted. + * @param successfulInsertions Documents that we successfully inserted. + * @param failedInsertions Documents that failed to be inserted, along with failure reason. */ public record InsertOperationPage( - List insertedIds, Map failedIds) + List allAttemptedInsertions, + boolean returnDocumentResponses, + List successfulInsertions, + List> failedInsertions) implements Supplier { + enum InsertionStatus { + OK, + ERROR, + SKIPPED + } + + @JsonPropertyOrder({"_id", "status", "errorsIdx"}) + @JsonInclude(JsonInclude.Include.NON_NULL) + record InsertionResult(DocumentId _id, InsertionStatus status, Integer errorsIdx) {} /** No-arg constructor, usually used for aggregation. */ - public InsertOperationPage() { - this(new ArrayList<>(), new HashMap<>()); + public InsertOperationPage( + List allAttemptedInsertions, + boolean returnDocumentResponses) { + this(allAttemptedInsertions, returnDocumentResponses, new ArrayList<>(), new ArrayList<>()); } /** {@inheritDoc} */ @Override public CommandResult get() { - // if we have errors, transform - if (null != failedIds && !failedIds().isEmpty()) { - - List errors = new ArrayList<>(failedIds.size()); - failedIds.forEach((documentId, throwable) -> errors.add(getError(documentId, throwable))); + // Ensure insertions and errors are in the input order (wrt unordered insertions), + // regardless of output format + Collections.sort(successfulInsertions); + if (!failedInsertions().isEmpty()) { + Collections.sort( + failedInsertions, Comparator.comparing(tuple -> tuple.getItem1().position())); + } + if (!returnDocumentResponses()) { // legacy output, limited to ids, error messages + List errors; + if (failedInsertions().isEmpty()) { + errors = null; + } else { + errors = + failedInsertions.stream() + .map(tuple -> getError(tuple.getItem1().document().id(), tuple.getItem2())) + .toList(); + } + // Old style, simple ids: + List insertedIds = + successfulInsertions.stream().map(docAndPos -> docAndPos.document().id()).toList(); return new CommandResult(null, Map.of(CommandStatus.INSERTED_IDS, insertedIds), errors); } - // id no errors, just inserted ids - return new CommandResult(Map.of(CommandStatus.INSERTED_IDS, insertedIds)); + // New style output: detailed responses. + InsertionResult[] results = new InsertionResult[allAttemptedInsertions().size()]; + List errors = new ArrayList<>(); + + // Results array filled in order: first successful insertions + for (InsertOperation.WritableDocAndPosition docAndPos : successfulInsertions) { + results[docAndPos.position()] = + new InsertionResult(docAndPos.document().id(), InsertionStatus.OK, null); + } + // Second: failed insertions + for (Tuple2 failed : failedInsertions) { + InsertOperation.WritableDocAndPosition docAndPos = failed.getItem1(); + Throwable throwable = failed.getItem2(); + CommandResult.Error error = getError(throwable); + + // We want to avoid adding the same error multiple times, so we keep track of the index: + // either one exists, use it; or if not, add it and use the new index. + int errorIdx = errors.indexOf(error); + if (errorIdx < 0) { // new non-dup error; add it + errorIdx = errors.size(); // will be appended at the end + errors.add(error); + } + results[docAndPos.position()] = + new InsertionResult(docAndPos.document().id(), InsertionStatus.ERROR, errorIdx); + } + // And third, if any, skipped insertions; those that were not attempted (f.ex due + // to failure for ordered inserts) + for (int i = 0; i < results.length; i++) { + if (null == results[i]) { + results[i] = + new InsertionResult( + allAttemptedInsertions.get(i).document().id(), InsertionStatus.SKIPPED, null); + } + } + return new CommandResult( + null, Map.of(CommandStatus.DOCUMENT_RESPONSES, Arrays.asList(results)), errors); } private static CommandResult.Error getError(DocumentId documentId, Throwable throwable) { String message = "Failed to insert document with _id %s: %s".formatted(documentId, throwable.getMessage()); - - Map fields = new HashMap<>(); - fields.put("exceptionClass", throwable.getClass().getSimpleName()); - if (throwable instanceof JsonApiException jae) { - fields.put("errorCode", jae.getErrorCode().name()); - } return ThrowableToErrorMapper.getMapperWithMessageFunction().apply(throwable, message); } + private static CommandResult.Error getError(Throwable throwable) { + return ThrowableToErrorMapper.getMapperWithMessageFunction() + .apply(throwable, throwable.getMessage()); + } + /** * Aggregates the result of the insert operation into this object. * - * @param id ID of the document that was inserted written. - * @param failure If not null, means an error occurred during the write. + * @param docWithPosition Document that was inserted (or failed to be inserted) + * @param failure If not null, means an error occurred during attempted insertion */ - public void aggregate(DocumentId id, Throwable failure) { - if (null != failure) { - failedIds.put(id, failure); + public void aggregate(InsertOperation.WritableDocAndPosition docWithPosition, Throwable failure) { + if (null == failure) { + successfulInsertions.add(docWithPosition); } else { - insertedIds.add(id); + failedInsertions.add(Tuple2.of(docWithPosition, failure)); } } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertManyCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertManyCommandResolver.java index cb61163b51..da9115b79c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertManyCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertManyCommandResolver.java @@ -1,11 +1,9 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl; -import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.impl.InsertManyCommand; import io.stargate.sgv2.jsonapi.service.operation.model.Operation; import io.stargate.sgv2.jsonapi.service.operation.model.impl.InsertOperation; -import io.stargate.sgv2.jsonapi.service.projection.IndexingProjector; import io.stargate.sgv2.jsonapi.service.resolver.model.CommandResolver; import io.stargate.sgv2.jsonapi.service.shredding.Shredder; import io.stargate.sgv2.jsonapi.service.shredding.model.WritableShreddedDocument; @@ -18,12 +16,10 @@ public class InsertManyCommandResolver implements CommandResolver { private final Shredder shredder; - private final ObjectMapper objectMapper; @Inject - public InsertManyCommandResolver(Shredder shredder, ObjectMapper objectMapper) { + public InsertManyCommandResolver(Shredder shredder) { this.shredder = shredder; - this.objectMapper = objectMapper; } @Override @@ -33,15 +29,13 @@ public Class getCommandClass() { @Override public Operation resolveCommand(CommandContext ctx, InsertManyCommand command) { - final IndexingProjector projection = ctx.indexingProjector(); final List shreddedDocuments = command.documents().stream().map(doc -> shredder.shred(ctx, doc, null)).toList(); - // resolve ordered InsertManyCommand.Options options = command.options(); + boolean ordered = (null != options) && options.ordered(); + boolean returnDocumentResponses = (null != options) && options.returnDocumentResponses(); - boolean ordered = null != options && Boolean.TRUE.equals(options.ordered()); - - return new InsertOperation(ctx, shreddedDocuments, ordered); + return new InsertOperation(ctx, shreddedDocuments, ordered, false, returnDocumentResponses); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertOneCommandResolver.java index dc63dd2d38..7cd66c5f3f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/InsertOneCommandResolver.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.service.resolver.model.impl; -import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.api.model.command.CommandContext; import io.stargate.sgv2.jsonapi.api.model.command.impl.InsertOneCommand; import io.stargate.sgv2.jsonapi.service.operation.model.Operation; @@ -16,12 +15,10 @@ public class InsertOneCommandResolver implements CommandResolver { private final Shredder shredder; - private final ObjectMapper objectMapper; @Inject - public InsertOneCommandResolver(Shredder shredder, ObjectMapper objectMapper) { + public InsertOneCommandResolver(Shredder shredder) { this.shredder = shredder; - this.objectMapper = objectMapper; } @Override diff --git a/src/main/offline/io/stargate/sgv2/jsonapi/api/model/command/impl/BeginOfflineSessionCommand.java b/src/main/offline/io/stargate/sgv2/jsonapi/api/model/command/impl/BeginOfflineSessionCommand.java index 2eeeab4fb2..a6fd902c39 100644 --- a/src/main/offline/io/stargate/sgv2/jsonapi/api/model/command/impl/BeginOfflineSessionCommand.java +++ b/src/main/offline/io/stargate/sgv2/jsonapi/api/model/command/impl/BeginOfflineSessionCommand.java @@ -193,7 +193,8 @@ private FileWriterParams buildFileWriterParams() { new CommandContext(this.namespace, this.createCollection.name()), List.of(), true, - true); + true, + false); String insertStatementCQL = insertOperation.buildInsertQuery(hasVector); return new FileWriterParams( this.namespace, diff --git a/src/main/offline/io/stargate/sgv2/jsonapi/service/resolver/model/impl/OfflineInsertManyCommandResolver.java b/src/main/offline/io/stargate/sgv2/jsonapi/service/resolver/model/impl/OfflineInsertManyCommandResolver.java index c62cc5eb23..c8bffa9010 100644 --- a/src/main/offline/io/stargate/sgv2/jsonapi/service/resolver/model/impl/OfflineInsertManyCommandResolver.java +++ b/src/main/offline/io/stargate/sgv2/jsonapi/service/resolver/model/impl/OfflineInsertManyCommandResolver.java @@ -39,8 +39,10 @@ public Operation resolveCommand(CommandContext ctx, OfflineInsertManyCommand com command.documents().stream().map(doc -> shredder.shred(ctx, doc, null)).toList(); // Offline insert is always ordered - boolean ordered = true; + final boolean ordered = true; + // and no need to return document positions + final boolean returnDocumentResponses = false; - return new InsertOperation(ctx, shreddedDocuments, ordered, true); + return new InsertOperation(ctx, shreddedDocuments, ordered, true, returnDocumentResponses); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java index 0717eed72c..ce03fac320 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java @@ -4,16 +4,7 @@ import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.fail; -import static org.hamcrest.Matchers.any; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; +import static org.hamcrest.Matchers.*; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,8 +21,10 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Arrays; import java.util.Map; import java.util.UUID; +import java.util.regex.Pattern; import org.apache.commons.lang3.RandomStringUtils; import org.bson.types.ObjectId; import org.junit.jupiter.api.AfterEach; @@ -49,6 +42,9 @@ public class InsertIntegrationTest extends AbstractCollectionIntegrationTestBase { private final ObjectMapper MAPPER = new ObjectMapper(); + private static final Pattern UUID_REGEX = + Pattern.compile("[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}"); + @AfterEach public void cleanUpData() { deleteAllDocuments(); @@ -1324,14 +1320,8 @@ public void ordered() { { "insertMany": { "documents": [ - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc5", - "username": "user5" - } + { "_id": "doc4", "username": "user4" }, + { "_id": "doc5", "username": "user5" } ], "options" : { "ordered" : true @@ -1348,56 +1338,31 @@ public void ordered() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("status.insertedIds", contains("doc4", "doc5")) + .body("status.insertedIds", is(Arrays.asList("doc4", "doc5"))) .body("data", is(nullValue())) .body("errors", is(nullValue())); - json = - """ - { - "countDocuments": { - } - } - """; - - given() - .headers(getHeaders()) - .contentType(ContentType.JSON) - .body(json) - .when() - .post(CollectionResource.BASE_PATH, namespaceName, collectionName) - .then() - .statusCode(200) - .body("status.count", is(2)) - .body("errors", is(nullValue())); + verifyDocCount(2); } @Test - public void orderedDuplicateIds() { + public void orderedReturnResponses() { + final String UUID_KEY = UUID.randomUUID().toString(); String json = - """ - { - "insertMany": { - "documents": [ - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc4", - "username": "user4_duplicate" - }, - { - "_id": "doc5", - "username": "user5" + """ + { + "insertMany": { + "documents": [ + { "_id": "doc1", "username": "user1" }, + { "_id": {"$uuid":"%s"}, "username": "user2" } + ], + "options" : { + "ordered": true, "returnDocumentResponses": true + } } - ], - "options" : { - "ordered" : true } - } - } - """; + """ + .formatted(UUID_KEY); given() .headers(getHeaders()) @@ -1407,72 +1372,55 @@ public void orderedDuplicateIds() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("status.insertedIds", contains("doc4")) .body("data", is(nullValue())) - .body("errors[0].message", startsWith("Failed to insert document with _id 'doc4'")) - .body("errors[0].errorCode", is("DOCUMENT_ALREADY_EXISTS")); - - json = - """ - { - "countDocuments": { - } - } - """; + .body("errors", is(nullValue())) + .body( + "status.documentResponses", + is( + Arrays.asList( + Map.of("status", "OK", "_id", "doc1"), + Map.of("status", "OK", "_id", Map.of("$uuid", UUID_KEY))))); - given() - .headers(getHeaders()) - .contentType(ContentType.JSON) - .body(json) - .when() - .post(CollectionResource.BASE_PATH, namespaceName, collectionName) - .then() - .statusCode(200) - .body("status.count", is(1)) - .body("errors", is(nullValue())); + verifyDocCount(2); } @Test - public void orderedDuplicateDocumentNoNamespace() { + public void orderedNoDocIdReturnResponses() { String json = """ - { - "insertMany": { - "documents": [ - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc5", - "username": "user5" - } - ], - "options" : { - "ordered" : true - } - } - } - """; + { + "insertMany": { + "documents": [ + { "username": "user1" }, + { "username": "user2" } + ], + "options" : { + "ordered": true, "returnDocumentResponses": true + } + } + } + """; given() .headers(getHeaders()) .contentType(ContentType.JSON) .body(json) .when() - .post(CollectionResource.BASE_PATH, "something_else", collectionName) + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("status.insertedIds", is(nullValue())) .body("data", is(nullValue())) - .body( - "errors[0].message", - startsWith("The provided namespace does not exist: something_else")) - .body("errors[0].exceptionClass", is("JsonApiException")); + .body("errors", is(nullValue())) + .body("status.insertedIds", is(nullValue())) + .body("status.failedDocuments", is(nullValue())) + // now tricky part: [0, ] check + .body("status.documentResponses", hasSize(2)) + .body("status.documentResponses[0].status", is("OK")) + .body("status.documentResponses[0]._id", matchesPattern(UUID_REGEX)) + .body("status.documentResponses[1].status", is("OK")) + .body("status.documentResponses[1]._id", matchesPattern(UUID_REGEX)); + + verifyDocCount(2); } @Test @@ -1482,14 +1430,8 @@ public void unordered() { { "insertMany": { "documents": [ - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc5", - "username": "user5" - } + { "_id": "doc4", "username": "user4" }, + { "_id": "doc5", "username": "user5" } ] } } @@ -1507,24 +1449,7 @@ public void unordered() { .body("data", is(nullValue())) .body("errors", is(nullValue())); - json = - """ - { - "countDocuments": { - } - } - """; - - given() - .headers(getHeaders()) - .contentType(ContentType.JSON) - .body(json) - .when() - .post(CollectionResource.BASE_PATH, namespaceName, collectionName) - .then() - .statusCode(200) - .body("status.count", is(2)) - .body("errors", is(nullValue())); + verifyDocCount(2); } @Test @@ -1534,18 +1459,9 @@ public void unorderedDuplicateIds() { { "insertMany": { "documents": [ - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc4", - "username": "user4" - }, - { - "_id": "doc5", - "username": "user5" - } + { "_id": "doc4", "username": "user4" }, + { "_id": "doc4", "username": "user4" }, + { "_id": "doc5", "username": "user5" } ], "options": { "ordered": false } } @@ -1565,24 +1481,7 @@ public void unorderedDuplicateIds() { .body("errors[0].message", startsWith("Failed to insert document with _id 'doc4'")) .body("errors[0].errorCode", is("DOCUMENT_ALREADY_EXISTS")); - json = - """ - { - "countDocuments": { - } - } - """; - - given() - .headers(getHeaders()) - .contentType(ContentType.JSON) - .body(json) - .when() - .post(CollectionResource.BASE_PATH, namespaceName, collectionName) - .then() - .statusCode(200) - .body("status.count", is(2)) - .body("errors", is(nullValue())); + verifyDocCount(2); } @Test @@ -1860,6 +1759,167 @@ public void tryInsertTooBigPayload() { @Nested @Order(7) class InsertManyFails { + @Test + public void orderedFailOnDups() { + String json = + """ + { + "insertMany": { + "documents": [ + { "_id": "doc4", "username": "user4" }, + { "_id": "doc4", "username": "user4_duplicate" }, + { "_id": "doc5", "username": "user5" + } + ], + "options" : { "ordered" : true } + } + } + """; + + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.insertedIds", is(Arrays.asList("doc4"))) + .body("data", is(nullValue())) + .body("errors", hasSize(1)) + .body("errors[0].errorCode", is("DOCUMENT_ALREADY_EXISTS")) + .body("errors[0].exceptionClass", is("JsonApiException")) + .body("errors[0].message", startsWith("Failed to insert document with _id 'doc4'")); + + verifyDocCount(1); + } + + @Test + public void orderedFailOnDupsReturnPositions() { + String json = + """ + { + "insertMany": { + "documents": [ + { "_id": "doc1", "username": "userA" }, + { "_id": "doc1", "username": "userB" }, + { "_id": "doc2", "username": "userC" + } + ], + "options" : { "ordered": true, "returnDocumentResponses": true } + } + } + """; + + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("data", is(nullValue())) + .body("errors", hasSize(1)) + .body("errors[0].errorCode", is("DOCUMENT_ALREADY_EXISTS")) + .body("errors[0].exceptionClass", is("JsonApiException")) + .body("errors[0].message", is("Document already exists with the given _id")) + .body("insertedIds", is(nullValue())) + .body("status.documentResponses", hasSize(3)) + .body("status.documentResponses[0]", is(Map.of("_id", "doc1", "status", "OK"))) + .body( + "status.documentResponses[1]", + is(Map.of("_id", "doc1", "status", "ERROR", "errorsIdx", 0))) + .body("status.documentResponses[2]", is(Map.of("_id", "doc2", "status", "SKIPPED"))); + + verifyDocCount(1); + } + + @Test + public void unorderedFailOnDups() { + String json = + """ + { + "insertMany": { + "documents": [ + { "_id": "doc4", "username": "user4" }, + { "_id": "doc4", "username": "user4_duplicate" }, + { "_id": "doc5", "username": "user5" }, + { "_id": "doc4", "username": "user4_duplicate_2" }, + { "_id": "doc5", "username": "user5_duplicate" }, + { "_id": "doc4", "username": "user4_duplicate_3" } + ], + "options" : { "ordered" : false } + } + } + """; + + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + // Insertions can occur in any order, so we can't predict which is first + // within the input list + .body("status.insertedIds", containsInAnyOrder("doc4", "doc5")) + .body("status.insertedIds", hasSize(2)) + .body("data", is(nullValue())) + // We have 4 failures, reported in order of input documents -- but note that + // inserts may be executed in different order! This means that the very first + // Document to insert may fail as duplicate if it was executed after another + // document in the list with that id + .body("errors", hasSize(4)) + .body("errors[0].errorCode", is("DOCUMENT_ALREADY_EXISTS")) + .body("errors[0].exceptionClass", is("JsonApiException")) + .body("errors[0].message", startsWith("Failed to insert document with _id")) + .body("errors[1].errorCode", is("DOCUMENT_ALREADY_EXISTS")) + .body("errors[1].exceptionClass", is("JsonApiException")) + .body("errors[1].message", startsWith("Failed to insert document with _id")) + .body("errors[2].errorCode", is("DOCUMENT_ALREADY_EXISTS")) + .body("errors[2].exceptionClass", is("JsonApiException")) + .body("errors[2].message", startsWith("Failed to insert document with _id")) + .body("errors[3].errorCode", is("DOCUMENT_ALREADY_EXISTS")) + .body("errors[3].exceptionClass", is("JsonApiException")) + .body("errors[3].message", startsWith("Failed to insert document with _id")); + + verifyDocCount(2); + } + + @Test + public void orderedFailBadNamespace() { + String json = + """ + { + "insertMany": { + "documents": [ + { "_id": "doc4", "username": "user4" }, + { "_id": "doc5", "username": "user5" } + ], + "options" : { "ordered" : true } + } + } + """; + + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, "something_else", collectionName) + .then() + .statusCode(200) + .body("status.insertedIds", is(nullValue())) + .body("data", is(nullValue())) + .body("errors", hasSize(1)) + .body("errors[0].exceptionClass", is("JsonApiException")) + .body( + "errors[0].message", + startsWith("The provided namespace does not exist: something_else")); + } + @Test public void insertManyWithTooManyDocuments() { ArrayNode docs = MAPPER.createArrayNode(); @@ -1897,6 +1957,7 @@ public void insertManyWithTooManyDocuments() { .then() .statusCode(200) .body("data", is(nullValue())) + .body("errors", hasSize(1)) .body("errors[0].errorCode", is("COMMAND_FIELD_INVALID")) .body("errors[0].exceptionClass", is("JsonApiException")) .body( @@ -1926,6 +1987,19 @@ public void checkInsertManyMetrics() { } } + private void verifyDocCount(int expDocs) { + given() + .headers(getHeaders()) + .contentType(ContentType.JSON) + .body(" { \"countDocuments\": { } }") + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.count", is(expDocs)) + .body("errors", is(nullValue())); + } + private JsonNode createBigDoc(String docId, int minDocSize) { // While it'd be cleaner to build JsonNode representation, checking for // size much more expensive so go low-tech instead diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationTest.java index e1fa5a7c70..062813f76c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/InsertOperationTest.java @@ -263,7 +263,7 @@ public void insertManyOrdered() throws Exception { CommandContext commandContext = createCommandContextWithCommandName("jsonDocsWrittenInsertManyCommand"); Supplier execute = - new InsertOperation(commandContext, List.of(shredDocument1, shredDocument2), true) + new InsertOperation(commandContext, List.of(shredDocument1, shredDocument2), true, false) .execute(dataApiRequestInfo, queryExecutor) .subscribe() .withSubscriber(UniAssertSubscriber.create()) @@ -454,7 +454,7 @@ public void insertManyUnordered() throws Exception { Supplier execute = new InsertOperation( - COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), false) + COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), false, false) .execute(dataApiRequestInfo, queryExecutor) .subscribe() .withSubscriber(UniAssertSubscriber.create()) @@ -531,7 +531,7 @@ public void failureOrdered() throws Exception { Supplier execute = new InsertOperation( - COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), true) + COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), true, false) .execute(dataApiRequestInfo, queryExecutor) .subscribe() .withSubscriber(UniAssertSubscriber.create()) @@ -616,7 +616,7 @@ public void failureOrderedLastFails() throws Exception { Supplier execute = new InsertOperation( - COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), true) + COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), true, false) .execute(dataApiRequestInfo, queryExecutor) .subscribe() .withSubscriber(UniAssertSubscriber.create()) @@ -702,7 +702,7 @@ public void failureUnorderedPartial() throws Exception { Supplier execute = new InsertOperation( - COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), false) + COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), false, false) .execute(dataApiRequestInfo, queryExecutor) .subscribe() .withSubscriber(UniAssertSubscriber.create()) @@ -790,7 +790,7 @@ public void failureUnorderedAll() throws Exception { Supplier execute = new InsertOperation( - COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), false) + COMMAND_CONTEXT_NON_VECTOR, List.of(shredDocument1, shredDocument2), false, false) .execute(dataApiRequestInfo, queryExecutor) .subscribe() .withSubscriber(UniAssertSubscriber.create())