diff --git a/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java b/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java index 3d908260c6..037fbfbe05 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java @@ -31,7 +31,7 @@ interface Fields { String VECTOR_INDEX_FUNCTION_NAME = "similarity_function"; /** Field name used in projection clause to get similarity score in response. */ - String VECTOR_FUNCTION_PROJECTION_FIELD = "$similarity"; + String VECTOR_FUNCTION_SIMILARITY_FIELD = "$similarity"; // Current definition of valid JSON API names: note that this only validates // characters, not length limits (nor empty nor "too long" allowed but validated diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java index 2d4fe8361a..699fd315ad 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java @@ -188,7 +188,9 @@ private Uni processUpdate( returnUpdatedDocument ? updatedDocument : originalDocument; // Some operations (findOneAndUpdate) define projection to apply to // result: - resultProjection.applyProjection(documentToReturn); + if (documentToReturn != null) { // null for some Operation tests + resultProjection.applyProjection(documentToReturn); + } } return new UpdatedDocument( writableShreddedDocument.id(), upsert, documentToReturn, null); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index 12edd62961..b1332553cb 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.projection; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCode; @@ -23,12 +24,6 @@ public class DocumentProjector { * No-op projector that does not modify documents. Considered "exclusion" projector since "no * exclusions" is conceptually what happens ("no inclusions" would drop all content) */ - private static final DocumentProjector DEFAULT_PROJECTOR = - new DocumentProjector(null, false, false); - - private static final DocumentProjector DEFAULT_PROJECTOR_WITH_SIMILARITY = - new DocumentProjector(null, false, true); - private static final DocumentProjector INCLUDE_ALL_PROJECTOR = new DocumentProjector(null, false, false); @@ -57,7 +52,18 @@ private DocumentProjector( } public static DocumentProjector defaultProjector() { - return DEFAULT_PROJECTOR; + return DefaultProjectorWrapper.defaultProjector(); + } + + public static DocumentProjector includeAllProjector() { + return INCLUDE_ALL_PROJECTOR; + } + + DocumentProjector withIncludeSimilarity(boolean includeSimilarityScore) { + if (this.includeSimilarityScore == includeSimilarityScore) { + return this; + } + return new DocumentProjector(rootLayer, inclusion, includeSimilarityScore); } public static DocumentProjector createFromDefinition(JsonNode projectionDefinition) { @@ -69,9 +75,9 @@ public static DocumentProjector createFromDefinition( // First special case: "simple" default projection if (projectionDefinition == null || projectionDefinition.isEmpty()) { if (includeSimilarity) { - return DEFAULT_PROJECTOR_WITH_SIMILARITY; + return DefaultProjectorWrapper.defaultProjectorWithSimilarity(); } - return DEFAULT_PROJECTOR; + return DefaultProjectorWrapper.defaultProjector(); } if (!projectionDefinition.isObject()) { throw new JsonApiException( @@ -125,6 +131,7 @@ public void applyProjection(JsonNode document) { } public void applyProjection(JsonNode document, Float similarityScore) { + Objects.requireNonNull(document, "Document to call 'applyProjection()' on must not be null"); // null -> either include-add or exclude-all; but logic may seem counter-intuitive if (rootLayer == null) { if (inclusion) { // exclude-all @@ -133,7 +140,7 @@ public void applyProjection(JsonNode document, Float similarityScore) { // In either case, we may need to add similarity score if present if (includeSimilarityScore && similarityScore != null) { ((ObjectNode) document) - .put(DocumentConstants.Fields.VECTOR_FUNCTION_PROJECTION_FIELD, similarityScore); + .put(DocumentConstants.Fields.VECTOR_FUNCTION_SIMILARITY_FIELD, similarityScore); } return; } @@ -144,7 +151,7 @@ public void applyProjection(JsonNode document, Float similarityScore) { } if (includeSimilarityScore && similarityScore != null) { ((ObjectNode) document) - .put(DocumentConstants.Fields.VECTOR_FUNCTION_PROJECTION_FIELD, similarityScore); + .put(DocumentConstants.Fields.VECTOR_FUNCTION_SIMILARITY_FIELD, similarityScore); } } @@ -165,6 +172,34 @@ public int hashCode() { return rootLayer.hashCode(); } + /** + * Due to the way projection is handled, we need to handle construction of default instance via + * separate class (to avoid cyclic dependency) + */ + static class DefaultProjectorWrapper { + /** + * Default projector that drops $vector and $vectorize fields but otherwise leaves document + * as-is. Constructed from empty definition (no inclusions/exclusions). + */ + private static final DocumentProjector DEFAULT_PROJECTOR; + + static { + ObjectNode emptyDef = new ObjectNode(JsonNodeFactory.instance); + DEFAULT_PROJECTOR = PathCollector.collectPaths(emptyDef, false).buildProjector(); + } + + private static final DocumentProjector DEFAULT_PROJECTOR_WITH_SIMILARITY = + DEFAULT_PROJECTOR.withIncludeSimilarity(true); + + public static DocumentProjector defaultProjector() { + return DEFAULT_PROJECTOR; + } + + public static DocumentProjector defaultProjectorWithSimilarity() { + return DEFAULT_PROJECTOR_WITH_SIMILARITY; + } + } + /** * Helper object used to traverse and collection inclusion/exclusion path definitions and verify * that there are only one or the other (except for doc id). Does not build data structures for @@ -177,7 +212,11 @@ private static class PathCollector { private int exclusions, inclusions; - private Boolean idInclusion = null; + private Boolean idInclusion; + + private Boolean $vectorInclusion; + + private Boolean $vectorizeInclusion; /** Whether similarity score is needed. */ private final boolean includeSimilarityScore; @@ -191,36 +230,36 @@ static PathCollector collectPaths(JsonNode def, boolean includeSimilarity) { } public DocumentProjector buildProjector() { - if (isDefaultProjection()) { - return defaultProjector(); - } - // One more thing: do we need to add document id? if (inclusions > 0) { // inclusion-based projection - // doc-id included unless explicitly excluded return new DocumentProjector( - ProjectionLayer.buildLayersNoOverlap(paths, slices, !Boolean.FALSE.equals(idInclusion)), + ProjectionLayer.buildLayersForProjection( + paths, + slices, + // doc-id included unless explicitly excluded + !Boolean.FALSE.equals(idInclusion), + // $vector only included if explicitly included + Boolean.TRUE.equals($vectorInclusion), + // $vectorize only included if explicitly included + Boolean.TRUE.equals($vectorizeInclusion)), true, includeSimilarityScore); } else { // exclusion-based - // doc-id excluded only if explicitly excluded return new DocumentProjector( - ProjectionLayer.buildLayersNoOverlap(paths, slices, Boolean.FALSE.equals(idInclusion)), + ProjectionLayer.buildLayersForProjection( + paths, + slices, + // doc-id excluded only if explicitly excluded + Boolean.FALSE.equals(idInclusion), + // $vector excluded unless explicitly included + !Boolean.TRUE.equals($vectorInclusion), + // $vectorize excluded unless explicitly included + !Boolean.TRUE.equals($vectorizeInclusion)), false, includeSimilarityScore); } } - /** - * Accessor to use for checking if collected paths indicate "empty" (no-operation) projection: - * if so, caller can avoid actual construction or evaluation. - */ - boolean isDefaultProjection() { - // Only the case if we have no non-doc-id inclusions/exclusions AND - // doc-id is included (by default or explicitly) - return paths.isEmpty() && slices.isEmpty() && !Boolean.FALSE.equals(idInclusion); - } - PathCollector collectFromObject(JsonNode ob, String parentPath) { var it = ob.fields(); while (it.hasNext()) { @@ -338,6 +377,10 @@ private void addSlice(String path, JsonNode sliceDef) { private void addExclusion(String path) { if (DocumentConstants.Fields.DOC_ID.equals(path)) { idInclusion = false; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(path)) { + $vectorInclusion = false; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD.equals(path)) { + $vectorizeInclusion = false; } else { // Must not mix exclusions and inclusions if (inclusions > 0) { @@ -356,6 +399,10 @@ private void addExclusion(String path) { private void addInclusion(String path) { if (DocumentConstants.Fields.DOC_ID.equals(path)) { idInclusion = true; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(path)) { + $vectorInclusion = true; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD.equals(path)) { + $vectorizeInclusion = true; } else { // Must not mix exclusions and inclusions if (exclusions > 0) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java index fd3350ce10..6e5b84a97a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java @@ -60,7 +60,7 @@ public static IndexingProjector createForIndexing(Set allowed, Set allowed, Set dotPaths, List slices, boolean addDocId) { - return buildLayers(dotPaths, slices, addDocId, true); + public static ProjectionLayer buildLayersForProjection( + Collection dotPaths, + List slices, + boolean addDocId, + boolean add$vector, + boolean add$vectorize) { + return buildLayers(dotPaths, slices, true, addDocId, add$vector, add$vectorize); } - public static ProjectionLayer buildLayersOverlapOk(Collection dotPaths) { - return buildLayers(dotPaths, Collections.emptyList(), false, false); + public static ProjectionLayer buildLayersForIndexing(Collection dotPaths) { + return buildLayers(dotPaths, Collections.emptyList(), false, false, false, false); } private static ProjectionLayer buildLayers( - Collection dotPaths, List slices, boolean addDocId, boolean failOnOverlap) { + Collection dotPaths, + List slices, + boolean failOnOverlap, + boolean addDocId, + boolean add$vector, + boolean add$vectorize) { // Root is always branch (not terminal): ProjectionLayer root = new ProjectionLayer("", false); for (String fullPath : dotPaths) { @@ -83,6 +92,20 @@ private static ProjectionLayer buildLayers( root, new String[] {DocumentConstants.Fields.DOC_ID}); } + if (add$vector) { + buildPath( + failOnOverlap, + DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD, + root, + new String[] {DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD}); + } + if (add$vectorize) { + buildPath( + failOnOverlap, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + root, + new String[] {DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD}); + } return root; } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java index df49504158..a60d878fec 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java @@ -59,7 +59,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteMany return FindOperation.unsorted( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null, operationsConfig.maxDocumentDeleteCount() + 1, operationsConfig.defaultPageSize(), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java index 841c9536eb..6931fbbf0e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java @@ -62,7 +62,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteOneC return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.KEY, objectMapper, vector); @@ -74,7 +74,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteOneC return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -88,7 +88,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteOneC return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.KEY, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java index 7d14c686ae..79af76f79c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java @@ -66,7 +66,7 @@ private FindOperation getFindOperation( return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -78,7 +78,7 @@ private FindOperation getFindOperation( return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -92,7 +92,7 @@ private FindOperation getFindOperation( return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java index 948f1383ee..e808879517 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java @@ -84,7 +84,7 @@ private FindOperation getFindOperation( return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -96,7 +96,7 @@ private FindOperation getFindOperation( return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -110,7 +110,7 @@ private FindOperation getFindOperation( return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java index 34ac8c729c..b9a6056931 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java @@ -86,7 +86,7 @@ private FindOperation getFindOperation( return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -99,8 +99,8 @@ private FindOperation getFindOperation( commandContext, logicalExpression, // 24-Mar-2023, tatu: Since we update the document, need to avoid modifications on - // read path, hence pass identity projector. - DocumentProjector.defaultProjector(), + // read path: + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -115,8 +115,8 @@ private FindOperation getFindOperation( commandContext, logicalExpression, // 24-Mar-2023, tatu: Since we update the document, need to avoid modifications on - // read path, hence pass identity projector. - DocumentProjector.defaultProjector(), + // read path: + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java index 7e2576dfa6..9a6cd05493 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java @@ -58,7 +58,7 @@ public Operation resolveCommand(CommandContext commandContext, UpdateManyCommand false, upsert, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), operationsConfig.maxDocumentUpdateCount(), operationsConfig.lwt().retries()); } @@ -68,7 +68,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateMany return FindOperation.unsorted( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null != command.options() ? command.options().pageState() : null, Integer.MAX_VALUE, operationsConfig.defaultPageSize(), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java index 0fa037501c..ccbbe18e86 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java @@ -61,7 +61,7 @@ public Operation resolveCommand(CommandContext commandContext, UpdateOneCommand false, upsert, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), 1, operationsConfig.lwt().retries()); } @@ -81,7 +81,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateOneC return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -93,7 +93,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateOneC return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -107,7 +107,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateOneC return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java index e1945f0bda..4ee562341c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java @@ -126,8 +126,7 @@ public void byIdAfterUpdate() { "name": "Joe", "age": 42, "enabled": true, - "value": -1, - "$vector" : [ 0.5, -0.25 ] + "value": -1 } """)) .body("status.matchedCount", is(1)) @@ -144,7 +143,6 @@ public void byIdBeforeUpdate() { "name": "Bob", "age": 77, "enabled": true, - "$vector" : [ 0.5, -0.25 ], "value": 3 } """; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index 7f4dc44be5..ca3fe2c5c2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -202,7 +202,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "1"} + "filter" : {"_id" : "1"}, + "projection": { "*": 1 } } } """; @@ -224,8 +225,8 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.documents[0]", jsonEquals(expected)) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected)); } // Test to verify vector embedding size can exceed general Array length limit @@ -243,7 +244,8 @@ public void insertBigVectorThenSearch() { """ { "find": { - "filter" : {"_id" : "bigVector1"} + "filter" : {"_id" : "bigVector1"}, + "projection": { "*": 1 } } } """) @@ -455,7 +457,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "2"} + "filter" : {"_id" : "2"}, + "projection": { "*": 1 } } } """; @@ -477,8 +480,8 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.documents[0]", jsonEquals(expected)) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected)); } } @@ -1054,6 +1057,7 @@ public void setOperation() { { "findOneAndUpdate": { "filter" : {"_id": "2"}, + "projection": { "*": 1 }, "update" : {"$set" : {"$vector" : [0.25, 0.25, 0.25, 0.25, 0.25]}}, "options" : {"returnDocument" : "after"} } @@ -1068,11 +1072,11 @@ public void setOperation() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document._id", is("2")) - .body("data.document.$vector", contains(0.25f, 0.25f, 0.25f, 0.25f, 0.25f)) + .body("errors", is(nullValue())) .body("status.matchedCount", is(1)) .body("status.modifiedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document._id", is("2")) + .body("data.document.$vector", contains(0.25f, 0.25f, 0.25f, 0.25f, 0.25f)); } @Test @@ -1097,11 +1101,11 @@ public void unsetOperation() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document._id", is("1")) - .body("data.document.$vector", is(nullValue())) + .body("errors", is(nullValue())) .body("status.matchedCount", is(1)) .body("status.modifiedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document._id", is("1")) + .body("data.document.$vector", is(nullValue())); } @Test @@ -1112,6 +1116,7 @@ public void setOnInsertOperation() { { "findOneAndUpdate": { "filter" : {"_id": "11"}, + "projection": { "*": 1 }, "update" : {"$setOnInsert" : {"$vector": [0.11, 0.22, 0.33, 0.44, 0.55]}}, "options" : {"returnDocument" : "after", "upsert": true} } @@ -1176,12 +1181,13 @@ public void setBigVectorOperation() { .contentType(ContentType.JSON) .body( """ - { - "find": { - "filter" : {"_id" : "bigVectorForSet"} - } - } - """) + { + "find": { + "filter" : {"_id" : "bigVectorForSet"}, + "projection": { "*": 1 } + } + } + """) .when() .post(CollectionResource.BASE_PATH, namespaceName, bigVectorCollectionName) .then() @@ -1198,6 +1204,7 @@ public void setBigVectorOperation() { { "findOneAndUpdate": { "filter" : {"_id": "bigVectorForSet"}, + "projection": { "*": 1 }, "update" : {"$set" : {"$vector" : [ %s ]}}, "options" : {"returnDocument" : "after"} } @@ -1228,7 +1235,8 @@ public void setBigVectorOperation() { """ { "find": { - "filter" : {"_id" : "bigVectorForSet"} + "filter" : {"_id" : "bigVectorForSet"}, + "projection": { "*": 1 } } } """) @@ -1332,6 +1340,7 @@ public void findOneAndReplace() { { "findOneAndReplace": { "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, + "projection": { "*": 1 }, "replacement" : {"_id" : "3", "username": "user3", "status" : false, "$vector" : [0.12, 0.05, 0.08, 0.32, 0.6]}, "options" : {"returnDocument" : "after"} } @@ -1399,7 +1408,8 @@ public void findOneAndReplaceWithBigVector() { """ { "find": { - "filter" : {"_id" : "bigVectorForFindReplace"} + "filter" : {"_id" : "bigVectorForFindReplace"}, + "projection": { "*": 1 } } } """) @@ -1419,6 +1429,7 @@ public void findOneAndReplaceWithBigVector() { { "findOneAndReplace": { "filter" : {"_id" : "bigVectorForFindReplace"}, + "projection": { "*": 1 }, "replacement" : {"_id" : "bigVectorForFindReplace", "$vector" : [ %s ]}, "options" : {"returnDocument" : "after"} } @@ -1434,12 +1445,12 @@ public void findOneAndReplaceWithBigVector() { .post(CollectionResource.BASE_PATH, namespaceName, bigVectorCollectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("status.matchedCount", is(1)) .body("status.modifiedCount", is(1)) .body("data.document._id", is("bigVectorForFindReplace")) .body("data.document.$vector", is(notNullValue())) - .body("data.document.$vector", hasSize(BIG_VECTOR_SIZE)) - .body("errors", is(nullValue())); + .body("data.document.$vector", hasSize(BIG_VECTOR_SIZE)); // and verify it was set to value with expected size given() @@ -1449,7 +1460,8 @@ public void findOneAndReplaceWithBigVector() { """ { "find": { - "filter" : {"_id" : "bigVectorForFindReplace"} + "filter" : {"_id" : "bigVectorForFindReplace"}, + "projection": { "*": 1 } } } """) @@ -1472,6 +1484,7 @@ public void findOneAndDelete() { """ { "findOneAndDelete": { + "projection": { "*": 1 }, "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]} } } @@ -1485,11 +1498,11 @@ public void findOneAndDelete() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) + .body("status.deletedCount", is(1)) .body("data.document._id", is("3")) - .body("data.document.$vector", is(notNullValue())) .body("data.document.name", is("Vision Vector Frame")) - .body("status.deletedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document.$vector", is(notNullValue())); } @Test @@ -1514,9 +1527,9 @@ public void deleteOne() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("status.deletedCount", is(1)) - .body("data", is(nullValue())) - .body("errors", is(nullValue())); + .body("data", is(nullValue())); // ensure find does not find the document json = @@ -1536,9 +1549,9 @@ public void deleteOne() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document", is(nullValue())) + .body("errors", is(nullValue())) .body("status", is(nullValue())) - .body("errors", is(nullValue())); + .body("data.document", is(nullValue())); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java index c722a5a96d..79db453fd9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java @@ -235,7 +235,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "1"} + "filter" : {"_id" : "1"}, + "projection": { "$vector": 1 } } } """; @@ -253,10 +254,10 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("data.documents[0]._id", is("1")) .body("data.documents[0].$vector", is(notNullValue())) - .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)) - .body("errors", is(nullValue())); + .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)); } @Test @@ -460,7 +461,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "2"} + "filter" : {"_id" : "2"}, + "projection": { "$vector": 1 } } } """; @@ -473,9 +475,9 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("data.documents[0]._id", is("2")) - .body("data.documents[0].$vector", is(notNullValue())) - .body("errors", is(nullValue())); + .body("data.documents[0].$vector", is(notNullValue())); } } @@ -663,9 +665,9 @@ public void happyPathWithInvalidData() { .then() .statusCode(200) .body("errors", hasSize(1)) - .body("errors[0].message", startsWith("$vectorize search clause needs to be text value")) .body("errors[0].errorCode", is("SHRED_BAD_VECTORIZE_VALUE")) - .body("errors[0].exceptionClass", is("JsonApiException")); + .body("errors[0].exceptionClass", is("JsonApiException")) + .body("errors[0].message", startsWith("$vectorize search clause needs to be text value")); } @Test @@ -675,6 +677,7 @@ public void vectorizeSortDenyAll() { """ { "find": { + "projection": { "$vector": 1, "$vectorize" : 1 }, "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"} } } @@ -696,8 +699,8 @@ public void vectorizeSortDenyAll() { .body("data.documents", hasSize(1)) .body("data.documents[0]._id", is("1")) .body("data.documents[0].$vector", is(notNullValue())) - .body("data.documents[0].$vectorize", is(notNullValue())) - .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)); + .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)) + .body("data.documents[0].$vectorize", is(notNullValue())); } } @@ -736,8 +739,8 @@ public void happyPath() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document._id", is("1")) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.document._id", is("1")); } @Test @@ -817,6 +820,7 @@ public void setOperation() { { "findOneAndUpdate": { "filter" : {"_id": "2"}, + "projection": { "$vector": 1 }, "update" : {"$set" : {"description" : "ChatGPT upgraded", "$vectorize" : "ChatGPT upgraded"}}, "options" : {"returnDocument" : "after"} } @@ -881,6 +885,7 @@ public void setOnInsertOperation() { { "findOneAndUpdate": { "filter" : {"_id": "11"}, + "projection": { "$vector": 1 }, "update" : {"$setOnInsert" : {"$vectorize": "New data updated"}}, "options" : {"returnDocument" : "after", "upsert": true} } @@ -1006,6 +1011,7 @@ public void findOneAndReplace() { """ { "findOneAndReplace": { + "projection": { "$vector": 1 }, "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"}, "replacement" : {"_id" : "1", "username": "user1", "status" : false, "description" : "Updating new data", "$vectorize" : "Updating new data"}, "options" : {"returnDocument" : "after"} @@ -1079,7 +1085,8 @@ public void findOneAndDelete() { """ { "findOneAndDelete": { - "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"} + "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"}, + "projection": { "*": 1 } } } """; @@ -1097,11 +1104,11 @@ public void findOneAndDelete() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) + .body("status.deletedCount", is(1)) .body("data.document._id", is("1")) - .body("data.document.$vector", is(notNullValue())) .body("data.document.name", is("Coded Cleats")) - .body("status.deletedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document.$vector", is(notNullValue())); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java index 195920b1db..4a012863bb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java @@ -2643,7 +2643,7 @@ public void vectorSearch() throws Exception { FindOperation.vsearch( VECTOR_COMMAND_CONTEXT, implicitAnd, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null, 2, 2, @@ -2711,7 +2711,7 @@ public void vectorSearchWithFilter() throws Exception { FindOperation.vsearchSingle( VECTOR_COMMAND_CONTEXT, implicitAnd, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, new float[] {0.25f, 0.25f, 0.25f, 0.25f}); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java index f8a5b04834..8c4c7ecc36 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java @@ -1721,7 +1721,7 @@ public void withUpsert() throws Exception { false, true, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), 20, 3); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java index 03301d472f..70b2ec23d5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java @@ -37,12 +37,12 @@ public void verifyNoEmptyPath() throws Exception { JsonNode def = objectMapper.readTree( """ - { "root" : - { "branch" : - { "": 1 } - } - } - """); + { "root" : + { "branch" : + { "": 1 } + } + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -56,11 +56,11 @@ public void verifyNoIncludeAfterExclude() throws Exception { JsonNode def = objectMapper.readTree( """ - { "excludeMe" : 0, - "excludeMeToo" : 0, - "include.me" : 1 - } - """); + { "excludeMe" : 0, + "excludeMeToo" : 0, + "include.me" : 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -74,11 +74,11 @@ public void verifyNoPathOverlap() throws Exception { JsonNode def = objectMapper.readTree( """ - { "branch" : 1, - "branch.x.leaf" : 1, - "include.me" : 1 - } - """); + { "branch" : 1, + "branch.x.leaf" : 1, + "include.me" : 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -90,11 +90,11 @@ public void verifyNoPathOverlap() throws Exception { JsonNode def2 = objectMapper.readTree( """ - { "a.y.leaf" : 1, - "a" : 1, - "value" : 1 - } - """); + { "a.y.leaf" : 1, + "a" : 1, + "value" : 1 + } + """); Throwable t2 = catchThrowable(() -> DocumentProjector.createFromDefinition(def2)); assertThat(t2) .isInstanceOf(JsonApiException.class) @@ -108,16 +108,16 @@ public void verifyNoExcludeAfterInclude() throws Exception { JsonNode def = objectMapper.readTree( """ - { "includeMe" : 1, - "misc" : { - "nested": { - "do" : true, - "dont" : false - } - }, - "includeMe2" : 1 - } - """); + { "includeMe" : 1, + "misc" : { + "nested": { + "do" : true, + "dont" : false + } + }, + "includeMe2" : 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -157,10 +157,10 @@ public void verifyNoDollarSimilarity() throws Exception { JsonNode def = objectMapper.readTree( """ - { "_id": 1, - "$similarity": 1 - } - """); + { "_id": 1, + "$similarity": 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -177,11 +177,11 @@ public void verifyNoUnknownOperators() throws Exception { JsonNode def = objectMapper.readTree( """ - { "include" : { - "$set" : 1 - } - } - """); + { "include" : { + "$set" : 1 + } + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -230,12 +230,12 @@ public void verifySliceDefinitionNumberOrArray() throws Exception { JsonNode def = objectMapper.readTree( """ - { - "include" : { - "$slice" : "text-not-accepted" - } - } - """); + { + "include" : { + "$slice" : "text-not-accepted" + } + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -265,73 +265,110 @@ public void verifySliceDefinitionToReturnPositive() throws Exception { } @Nested - class ProjectorApplyInclusions { + class ProjectorApplyDefaultProjection { // [json-api#634]: empty Object same as "include all" @Test - public void testIncludeWithEmptyProject() throws Exception { - final JsonNode doc = - objectMapper.readTree( - """ - { - "_id" : 1, - "value1": 42 - } - """); + public void defaultProjectionRegularFieldsOnly() throws Exception { + final String docJson = + """ + { + "_id" : 1, + "value1": 42 + } + """; DocumentProjector projection = DocumentProjector.createFromDefinition(objectMapper.readTree("{ }")); + final JsonNode doc = objectMapper.readTree(docJson); // Technically considered "Exclusion" but one that excludes nothing assertThat(projection.isInclusion()).isFalse(); projection.applyProjection(doc); - assertThat(doc).isEqualTo(doc); + assertThat(doc).isEqualTo(objectMapper.readTree(docJson)); } + @Test + public void defaultProjectionMixAll() throws Exception { + final String docJson = + """ + { + "_id" : 1, + "value1": 42, + "$vectorize": "Quick brown fox", + "$vector": [0.0, 1.0], + "value2": -3 + } + """; + DocumentProjector projection = + DocumentProjector.createFromDefinition(objectMapper.readTree("{ }")); + final JsonNode doc = objectMapper.readTree(docJson); + + assertThat(projection.isInclusion()).isFalse(); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id" : 1, + "value1": 42, + "value2": -3 + } + """)); + } + } + + @Nested + class ProjectorApplyInclusions { + @Test public void testSimpleIncludeWithId() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 1, - "value1" : true, - "value2" : false, - "nested" : { - "x": 3, - "y": 4, - "z": -1 - }, - "nested2" : { - "z": 5 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """); + { "_id" : 1, + "value1" : true, + "value2" : false, + "nested" : { + "x": 3, + "y": 4, + "z": -1 + }, + "nested2" : { + "z": 5 + }, + "$vectorize": "hello work!", + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { "value2" : 1, - "nested" : { - "x": 1 - }, - "nested.z": 1, - "nosuchprop": 1, - "$vector": 1 - } - """)); + { "value2" : 1, + "nested" : { + "x": 1 + }, + "nested.z": 1, + "nosuchprop": 1, + "$vector": 1, + "$vectorize": 1 + } + """)); assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { "_id" : 1, - "value2" : false, - "nested" : { - "x": 3, - "z": -1 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """)); + { "_id" : 1, + "value2" : false, + "nested" : { + "x": 3, + "z": -1 + }, + "$vectorize": "hello work!", + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """)); } @Test @@ -339,33 +376,36 @@ public void testSimpleIncludeWithSimilarity() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 1, - "value1" : true, - "value2" : false, - "nested" : { - "x": 3, - "y": 4, - "z": -1 - }, - "nested2" : { - "z": 5 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """); + { "_id" : 1, + "value1" : true, + "value2" : false, + "nested" : { + "x": 3, + "y": 4, + "z": -1 + }, + "nested2" : { + "z": 5 + }, + "$vectorize": "hello work!", + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { "value2" : 1, - "$vector": 1 - } - """), + { "value2" : 1, + "$vector": 1 + } + """), true); assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc, 0.25f); assertThat(doc.get("value2")).isNotNull(); + // we only include $vector explicitly, not $vectorize so: assertThat(doc.get("$vector")).isNotNull(); + assertThat(doc.get("$vectorize")).isNull(); assertThat(doc.get("$similarity").floatValue()).isEqualTo(0.25f); } @@ -374,43 +414,43 @@ public void testSimpleIncludeWithoutId() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 1, - "value1" : true, - "nested" : { - "x": 3, - "z": -1 - }, - "nested2" : { - "z": 5 - } - } - """); + { "_id" : 1, + "value1" : true, + "nested" : { + "x": 3, + "z": -1 + }, + "nested2" : { + "z": 5 + } + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { "value1" : 1, - "nested" : { - "x": 1 - }, - "_id": 0, - "nested2.unknown": 1 - } - """)); + { "value1" : 1, + "nested" : { + "x": 1 + }, + "_id": 0, + "nested2.unknown": 1 + } + """)); assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { - "value1": true, - "nested" : { - "x": 3 - }, - "nested2" : { } - } - """)); + { + "value1": true, + "nested" : { + "x": 3 + }, + "nested2" : { } + } + """)); } @Test @@ -418,17 +458,17 @@ public void testSimpleIncludeInArray() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "values" : [ { - "x": 1, - "y": 2 - }, { - "y": false, - "z": true - } ], - "array2": [1, 2], - "array3": [2, 3] - } - """); + { "values" : [ { + "x": 1, + "y": 2 + }, { + "y": false, + "z": true + } ], + "array2": [1, 2], + "array3": [2, 3] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree("{ \"values.y\": 1, \"values.z\":1, \"array3\":1}")); @@ -438,15 +478,15 @@ public void testSimpleIncludeInArray() throws Exception { .isEqualTo( objectMapper.readTree( """ - { "values" : [ { - "y": 2 - }, { - "y": false, - "z": true - } ], - "array3": [2, 3] - } - """)); + { "values" : [ { + "y": 2 + }, { + "y": false, + "z": true + } ], + "array3": [2, 3] + } + """)); } } @@ -457,51 +497,51 @@ public void excludeWithIdIncluded() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 123, - "value1" : true, - "value2" : false, - "nested" : { - "x": 3, - "y": 4, - "z": -1 - }, - "nested2" : { - "z": 5 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """); + { "_id" : 123, + "value1" : true, + "value2" : false, + "nested" : { + "x": 3, + "y": 4, + "z": -1 + }, + "nested2" : { + "z": 5 + }, + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { - "value1" : 0, - "nested" : { - "x": 0 - }, - "nested.z": 0, - "nosuchprop": 0, - "$vector": 0 - } - """)); + { + "value1" : 0, + "nested" : { + "x": 0 + }, + "nested.z": 0, + "nosuchprop": 0, + "$vector": 0 + } + """)); assertThat(projection.isInclusion()).isFalse(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { - "_id" : 123, - "value2" : false, - "nested" : { - "y": 4 - }, - "nested2" : { - "z": 5 - } - } - """)); + { + "_id" : 123, + "value2" : false, + "nested" : { + "y": 4 + }, + "nested2" : { + "z": 5 + } + } + """)); } @Test @@ -509,45 +549,45 @@ public void excludeWithIdExcluded() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 123, - "value1" : true, - "nested" : { - "x": 3, - "z": -1 - }, - "nested2" : { - "z": 5 - } - } - """); + { "_id" : 123, + "value1" : true, + "nested" : { + "x": 3, + "z": -1 + }, + "nested2" : { + "z": 5 + } + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { - "_id": 0, - "value1" : 0, - "nested" : { - "x": 0 - }, - "nested2.unknown": 0 - } - """)); + { + "_id": 0, + "value1" : 0, + "nested" : { + "x": 0 + }, + "nested2.unknown": 0 + } + """)); assertThat(projection.isInclusion()).isFalse(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { - "nested" : { - "z": -1 - }, - "nested2" : { - "z" : 5 - } - } - """)); + { + "nested" : { + "z": -1 + }, + "nested2" : { + "z" : 5 + } + } + """)); } @Test @@ -555,17 +595,17 @@ public void excludeInArray() throws Exception { JsonNode doc = objectMapper.readTree( """ - { "values" : [ { - "x": 1, - "y": 2 - }, { - "y": false, - "z": true - } ], - "array2": [2, 3], - "array3": [2, 3] - } - """); + { "values" : [ { + "x": 1, + "y": 2 + }, { + "y": false, + "z": true + } ], + "array2": [2, 3], + "array3": [2, 3] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree("{ \"values.y\": 0, \"values.z\":0,\"array3\":0}")); @@ -575,13 +615,13 @@ public void excludeInArray() throws Exception { .isEqualTo( objectMapper.readTree( """ - { "values" : [ { - "x": 1 - }, { - } ], - "array2": [2, 3] - } - """)); + { "values" : [ { + "x": 1 + }, { + } ], + "array2": [2, 3] + } + """)); } @Test @@ -589,18 +629,18 @@ public void excludeInSubDoc() throws Exception { JsonNode doc = objectMapper.readTree( """ - { - "_id": "doc5", - "username": "user5", - "sub_doc" : { - "a": 5, - "b": { - "c": "v1", - "d": false - } - } - } - """); + { + "_id": "doc5", + "username": "user5", + "sub_doc" : { + "a": 5, + "b": { + "c": "v1", + "d": false + } + } + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition(objectMapper.readTree("{ \"sub_doc.b\": 0 }")); assertThat(projection.isInclusion()).isFalse(); @@ -609,14 +649,14 @@ public void excludeInSubDoc() throws Exception { .isEqualTo( objectMapper.readTree( """ - { - "_id": "doc5", - "username": "user5", - "sub_doc" : { - "a": 5 - } - } - """)); + { + "_id": "doc5", + "username": "user5", + "sub_doc" : { + "a": 5 + } + } + """)); } // "Empty" Projection is not really inclusion or exclusion, but technically @@ -640,6 +680,253 @@ public void emptyProjectionAsExclude() throws Exception { } } + // Tests to see that specific handling of _id works with various + // configurations + @Nested + class ProjectorApplyIdExcludeInclude { + @Test + void includeIdExcludeProperty() throws Exception { + final String docJson = + """ + { + "_id": "id", + "value1": 1, + "value2": 2, + "value3": 3 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 1, "value2": 0 } + """)); + // exclusion since we have explicit exclusion for non-id field + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value1": 1, + "value3": 3 + } + """)); + + // Then the other way around + projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "value2": 0, "_id": 1 } + """)); + // exclusion since we have explicit exclusion for non-id field + assertThat(projection.isInclusion()).isFalse(); + doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value1": 1, + "value3": 3 + } + """)); + } + + @Test + void excludeIdIncludeProperty() throws Exception { + final String docJson = + """ + { + "_id": "id", + "value1": 1, + "value2": 2, + "value3": 3 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 0, "value2": 1 } + """)); + // inclusion since we have explicit inclusion for non-id field + assertThat(projection.isInclusion()).isTrue(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "value2": 2 + } + """)); + + // then reverse order for filter + projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "value2": 1, "_id": 0 } + """)); + // inclusion since we have explicit inclusion for non-id field + assertThat(projection.isInclusion()).isTrue(); + doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "value2": 2 + } + """)); + } + + @Test + void includeIdExcludeVector() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vector": [0.25, 0.5], + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 1, "$vector": 0 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value": 42 + } + """)); + } + + @Test + void excludeIdIncludeVector() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vector": [0.25, 0.5], + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 0, "$vector": 1 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "$vector": [0.25, 0.5], + "value": 42 + } + """)); + } + + @Test + void includeIdExcludeVectorize() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vectorize": "Hello world!", + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 1, "$vectorize": 0 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value": 42 + } + """)); + } + + @Test + void excludeIdIncludeVectorize() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vectorize": "Hello world!", + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 0, "$vectorize": 1 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "$vectorize": "Hello world!", + "value": 42 + } + """)); + } + } + // Special case of [data-api#1001]: include-all / exclude-all @Nested class ProjectorApplyStarIncludeOrExclude { @@ -650,6 +937,7 @@ public void includeAll() throws Exception { { "_id": "docStarInclude", "value": 42, + "$vectorize": "Quick brown fox", "$vector": [ 1.0, 0.5, -0.25 ] } """; @@ -684,6 +972,7 @@ public void excludeAll() throws Exception { { "_id": "docStarExclude", "value": 42, + "$vectorize": "Quick brown fox", "$vector": [ 1.0, 0.5, -0.25 ] } """;