From 28008e5ba05802d359b3e7fc7f95ec4321d47347 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 2 Apr 2024 15:14:44 -0700 Subject: [PATCH 01/17] Add support for default exclusion of $vector --- .../jsonapi/service/projection/DocumentProjector.java | 6 ++++-- .../jsonapi/service/projection/IndexingProjector.java | 6 +++--- .../sgv2/jsonapi/service/projection/ProjectionLayer.java | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index 12edd62961..35cf188a30 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -199,13 +199,15 @@ public DocumentProjector buildProjector() { if (inclusions > 0) { // inclusion-based projection // doc-id included unless explicitly excluded return new DocumentProjector( - ProjectionLayer.buildLayersNoOverlap(paths, slices, !Boolean.FALSE.equals(idInclusion)), + ProjectionLayer.buildLayersForProjection( + paths, slices, !Boolean.FALSE.equals(idInclusion)), true, includeSimilarityScore); } else { // exclusion-based // doc-id excluded only if explicitly excluded return new DocumentProjector( - ProjectionLayer.buildLayersNoOverlap(paths, slices, Boolean.FALSE.equals(idInclusion)), + ProjectionLayer.buildLayersForProjection( + paths, slices, Boolean.FALSE.equals(idInclusion)), false, includeSimilarityScore); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java index fd3350ce10..6e5b84a97a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/IndexingProjector.java @@ -60,7 +60,7 @@ public static IndexingProjector createForIndexing(Set allowed, Set allowed, Set dotPaths, List slices, boolean addDocId) { - return buildLayers(dotPaths, slices, addDocId, true); + return buildLayers(dotPaths, slices, true, addDocId); } - public static ProjectionLayer buildLayersOverlapOk(Collection dotPaths) { + public static ProjectionLayer buildLayersForIndexing(Collection dotPaths) { return buildLayers(dotPaths, Collections.emptyList(), false, false); } private static ProjectionLayer buildLayers( - Collection dotPaths, List slices, boolean addDocId, boolean failOnOverlap) { + Collection dotPaths, List slices, boolean failOnOverlap, boolean addDocId) { // Root is always branch (not terminal): ProjectionLayer root = new ProjectionLayer("", false); for (String fullPath : dotPaths) { From 75e3489d93ea737523719c0903c17253e685a118 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 2 Apr 2024 15:49:23 -0700 Subject: [PATCH 02/17] Add _id tests for DocumentProjector (for cases not yet covered) --- .../projection/DocumentProjectorTest.java | 605 +++++++++++------- 1 file changed, 360 insertions(+), 245 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java index 03301d472f..a859b182a9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java @@ -37,12 +37,12 @@ public void verifyNoEmptyPath() throws Exception { JsonNode def = objectMapper.readTree( """ - { "root" : - { "branch" : - { "": 1 } - } - } - """); + { "root" : + { "branch" : + { "": 1 } + } + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -56,11 +56,11 @@ public void verifyNoIncludeAfterExclude() throws Exception { JsonNode def = objectMapper.readTree( """ - { "excludeMe" : 0, - "excludeMeToo" : 0, - "include.me" : 1 - } - """); + { "excludeMe" : 0, + "excludeMeToo" : 0, + "include.me" : 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -74,11 +74,11 @@ public void verifyNoPathOverlap() throws Exception { JsonNode def = objectMapper.readTree( """ - { "branch" : 1, - "branch.x.leaf" : 1, - "include.me" : 1 - } - """); + { "branch" : 1, + "branch.x.leaf" : 1, + "include.me" : 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -90,11 +90,11 @@ public void verifyNoPathOverlap() throws Exception { JsonNode def2 = objectMapper.readTree( """ - { "a.y.leaf" : 1, - "a" : 1, - "value" : 1 - } - """); + { "a.y.leaf" : 1, + "a" : 1, + "value" : 1 + } + """); Throwable t2 = catchThrowable(() -> DocumentProjector.createFromDefinition(def2)); assertThat(t2) .isInstanceOf(JsonApiException.class) @@ -108,16 +108,16 @@ public void verifyNoExcludeAfterInclude() throws Exception { JsonNode def = objectMapper.readTree( """ - { "includeMe" : 1, - "misc" : { - "nested": { - "do" : true, - "dont" : false - } - }, - "includeMe2" : 1 - } - """); + { "includeMe" : 1, + "misc" : { + "nested": { + "do" : true, + "dont" : false + } + }, + "includeMe2" : 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -157,10 +157,10 @@ public void verifyNoDollarSimilarity() throws Exception { JsonNode def = objectMapper.readTree( """ - { "_id": 1, - "$similarity": 1 - } - """); + { "_id": 1, + "$similarity": 1 + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -177,11 +177,11 @@ public void verifyNoUnknownOperators() throws Exception { JsonNode def = objectMapper.readTree( """ - { "include" : { - "$set" : 1 - } - } - """); + { "include" : { + "$set" : 1 + } + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -230,12 +230,12 @@ public void verifySliceDefinitionNumberOrArray() throws Exception { JsonNode def = objectMapper.readTree( """ - { - "include" : { - "$slice" : "text-not-accepted" - } - } - """); + { + "include" : { + "$slice" : "text-not-accepted" + } + } + """); Throwable t = catchThrowable(() -> DocumentProjector.createFromDefinition(def)); assertThat(t) .isInstanceOf(JsonApiException.class) @@ -290,48 +290,48 @@ public void testSimpleIncludeWithId() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 1, - "value1" : true, - "value2" : false, - "nested" : { - "x": 3, - "y": 4, - "z": -1 - }, - "nested2" : { - "z": 5 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """); + { "_id" : 1, + "value1" : true, + "value2" : false, + "nested" : { + "x": 3, + "y": 4, + "z": -1 + }, + "nested2" : { + "z": 5 + }, + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { "value2" : 1, - "nested" : { - "x": 1 - }, - "nested.z": 1, - "nosuchprop": 1, - "$vector": 1 - } - """)); + { "value2" : 1, + "nested" : { + "x": 1 + }, + "nested.z": 1, + "nosuchprop": 1, + "$vector": 1 + } + """)); assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { "_id" : 1, - "value2" : false, - "nested" : { - "x": 3, - "z": -1 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """)); + { "_id" : 1, + "value2" : false, + "nested" : { + "x": 3, + "z": -1 + }, + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """)); } @Test @@ -339,28 +339,28 @@ public void testSimpleIncludeWithSimilarity() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 1, - "value1" : true, - "value2" : false, - "nested" : { - "x": 3, - "y": 4, - "z": -1 - }, - "nested2" : { - "z": 5 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """); + { "_id" : 1, + "value1" : true, + "value2" : false, + "nested" : { + "x": 3, + "y": 4, + "z": -1 + }, + "nested2" : { + "z": 5 + }, + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { "value2" : 1, - "$vector": 1 - } - """), + { "value2" : 1, + "$vector": 1 + } + """), true); assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc, 0.25f); @@ -374,43 +374,43 @@ public void testSimpleIncludeWithoutId() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 1, - "value1" : true, - "nested" : { - "x": 3, - "z": -1 - }, - "nested2" : { - "z": 5 - } - } - """); + { "_id" : 1, + "value1" : true, + "nested" : { + "x": 3, + "z": -1 + }, + "nested2" : { + "z": 5 + } + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { "value1" : 1, - "nested" : { - "x": 1 - }, - "_id": 0, - "nested2.unknown": 1 - } - """)); + { "value1" : 1, + "nested" : { + "x": 1 + }, + "_id": 0, + "nested2.unknown": 1 + } + """)); assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { - "value1": true, - "nested" : { - "x": 3 - }, - "nested2" : { } - } - """)); + { + "value1": true, + "nested" : { + "x": 3 + }, + "nested2" : { } + } + """)); } @Test @@ -418,17 +418,17 @@ public void testSimpleIncludeInArray() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "values" : [ { - "x": 1, - "y": 2 - }, { - "y": false, - "z": true - } ], - "array2": [1, 2], - "array3": [2, 3] - } - """); + { "values" : [ { + "x": 1, + "y": 2 + }, { + "y": false, + "z": true + } ], + "array2": [1, 2], + "array3": [2, 3] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree("{ \"values.y\": 1, \"values.z\":1, \"array3\":1}")); @@ -438,15 +438,15 @@ public void testSimpleIncludeInArray() throws Exception { .isEqualTo( objectMapper.readTree( """ - { "values" : [ { - "y": 2 - }, { - "y": false, - "z": true - } ], - "array3": [2, 3] - } - """)); + { "values" : [ { + "y": 2 + }, { + "y": false, + "z": true + } ], + "array3": [2, 3] + } + """)); } } @@ -457,51 +457,51 @@ public void excludeWithIdIncluded() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 123, - "value1" : true, - "value2" : false, - "nested" : { - "x": 3, - "y": 4, - "z": -1 - }, - "nested2" : { - "z": 5 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """); + { "_id" : 123, + "value1" : true, + "value2" : false, + "nested" : { + "x": 3, + "y": 4, + "z": -1 + }, + "nested2" : { + "z": 5 + }, + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { - "value1" : 0, - "nested" : { - "x": 0 - }, - "nested.z": 0, - "nosuchprop": 0, - "$vector": 0 - } - """)); + { + "value1" : 0, + "nested" : { + "x": 0 + }, + "nested.z": 0, + "nosuchprop": 0, + "$vector": 0 + } + """)); assertThat(projection.isInclusion()).isFalse(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { - "_id" : 123, - "value2" : false, - "nested" : { - "y": 4 - }, - "nested2" : { - "z": 5 - } - } - """)); + { + "_id" : 123, + "value2" : false, + "nested" : { + "y": 4 + }, + "nested2" : { + "z": 5 + } + } + """)); } @Test @@ -509,45 +509,45 @@ public void excludeWithIdExcluded() throws Exception { final JsonNode doc = objectMapper.readTree( """ - { "_id" : 123, - "value1" : true, - "nested" : { - "x": 3, - "z": -1 - }, - "nested2" : { - "z": 5 - } - } - """); + { "_id" : 123, + "value1" : true, + "nested" : { + "x": 3, + "z": -1 + }, + "nested2" : { + "z": 5 + } + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree( """ - { - "_id": 0, - "value1" : 0, - "nested" : { - "x": 0 - }, - "nested2.unknown": 0 - } - """)); + { + "_id": 0, + "value1" : 0, + "nested" : { + "x": 0 + }, + "nested2.unknown": 0 + } + """)); assertThat(projection.isInclusion()).isFalse(); projection.applyProjection(doc); assertThat(doc) .isEqualTo( objectMapper.readTree( """ - { - "nested" : { - "z": -1 - }, - "nested2" : { - "z" : 5 - } - } - """)); + { + "nested" : { + "z": -1 + }, + "nested2" : { + "z" : 5 + } + } + """)); } @Test @@ -555,17 +555,17 @@ public void excludeInArray() throws Exception { JsonNode doc = objectMapper.readTree( """ - { "values" : [ { - "x": 1, - "y": 2 - }, { - "y": false, - "z": true - } ], - "array2": [2, 3], - "array3": [2, 3] - } - """); + { "values" : [ { + "x": 1, + "y": 2 + }, { + "y": false, + "z": true + } ], + "array2": [2, 3], + "array3": [2, 3] + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition( objectMapper.readTree("{ \"values.y\": 0, \"values.z\":0,\"array3\":0}")); @@ -575,13 +575,13 @@ public void excludeInArray() throws Exception { .isEqualTo( objectMapper.readTree( """ - { "values" : [ { - "x": 1 - }, { - } ], - "array2": [2, 3] - } - """)); + { "values" : [ { + "x": 1 + }, { + } ], + "array2": [2, 3] + } + """)); } @Test @@ -589,18 +589,18 @@ public void excludeInSubDoc() throws Exception { JsonNode doc = objectMapper.readTree( """ - { - "_id": "doc5", - "username": "user5", - "sub_doc" : { - "a": 5, - "b": { - "c": "v1", - "d": false - } - } - } - """); + { + "_id": "doc5", + "username": "user5", + "sub_doc" : { + "a": 5, + "b": { + "c": "v1", + "d": false + } + } + } + """); DocumentProjector projection = DocumentProjector.createFromDefinition(objectMapper.readTree("{ \"sub_doc.b\": 0 }")); assertThat(projection.isInclusion()).isFalse(); @@ -609,14 +609,14 @@ public void excludeInSubDoc() throws Exception { .isEqualTo( objectMapper.readTree( """ - { - "_id": "doc5", - "username": "user5", - "sub_doc" : { - "a": 5 - } - } - """)); + { + "_id": "doc5", + "username": "user5", + "sub_doc" : { + "a": 5 + } + } + """)); } // "Empty" Projection is not really inclusion or exclusion, but technically @@ -640,6 +640,121 @@ public void emptyProjectionAsExclude() throws Exception { } } + // Tests to see that specific handling of _id works with various + // configurations + @Nested + class ProjectorApplyIdExcludeInclude { + @Test + void includeIdExcludeProperty() throws Exception { + final String docJson = + """ + { + "_id": "id", + "value1": 1, + "value2": 2, + "value3": 3 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 1, "value2": 0 } + """)); + // exclusion since we have explicit exclusion for non-id field + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value1": 1, + "value3": 3 + } + """)); + + // Then the other way around + projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "value2": 0, "_id": 1 } + """)); + // exclusion since we have explicit exclusion for non-id field + assertThat(projection.isInclusion()).isFalse(); + doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value1": 1, + "value3": 3 + } + """)); + } + + @Test + void excludeIdIncludeProperty() throws Exception { + final String docJson = + """ + { + "_id": "id", + "value1": 1, + "value2": 2, + "value3": 3 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 0, "value2": 1 } + """)); + // inclusion since we have explicit inclusion for non-id field + assertThat(projection.isInclusion()).isTrue(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "value2": 2 + } + """)); + + // then reverse order for filter + projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "value2": 1, "_id": 0 } + """)); + // inclusion since we have explicit inclusion for non-id field + assertThat(projection.isInclusion()).isTrue(); + doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "value2": 2 + } + """)); + } + } + // Special case of [data-api#1001]: include-all / exclude-all @Nested class ProjectorApplyStarIncludeOrExclude { From 29041209a583dbeea4abf88a49bd25db9f877842 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 2 Apr 2024 16:02:04 -0700 Subject: [PATCH 03/17] First part of change to support $vector exclusion --- .../service/projection/DocumentProjector.java | 22 +++++++++++++++---- .../service/projection/ProjectionLayer.java | 19 ++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index 35cf188a30..b659c924ac 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -179,6 +179,8 @@ private static class PathCollector { private Boolean idInclusion = null; + private Boolean $vectorInclusion = null; + /** Whether similarity score is needed. */ private final boolean includeSimilarityScore; @@ -197,17 +199,25 @@ public DocumentProjector buildProjector() { // One more thing: do we need to add document id? if (inclusions > 0) { // inclusion-based projection - // doc-id included unless explicitly excluded return new DocumentProjector( ProjectionLayer.buildLayersForProjection( - paths, slices, !Boolean.FALSE.equals(idInclusion)), + paths, + slices, + // doc-id included unless explicitly excluded + !Boolean.FALSE.equals(idInclusion), + // $vector only included if explicitly included + Boolean.TRUE.equals($vectorInclusion)), true, includeSimilarityScore); } else { // exclusion-based - // doc-id excluded only if explicitly excluded return new DocumentProjector( ProjectionLayer.buildLayersForProjection( - paths, slices, Boolean.FALSE.equals(idInclusion)), + paths, + slices, + // doc-id excluded only if explicitly excluded + Boolean.FALSE.equals(idInclusion), + // $vector excluded unless explicitly included + !Boolean.TRUE.equals($vectorInclusion)), false, includeSimilarityScore); } @@ -340,6 +350,8 @@ private void addSlice(String path, JsonNode sliceDef) { private void addExclusion(String path) { if (DocumentConstants.Fields.DOC_ID.equals(path)) { idInclusion = false; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(path)) { + $vectorInclusion = false; } else { // Must not mix exclusions and inclusions if (inclusions > 0) { @@ -358,6 +370,8 @@ private void addExclusion(String path) { private void addInclusion(String path) { if (DocumentConstants.Fields.DOC_ID.equals(path)) { idInclusion = true; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(path)) { + $vectorInclusion = true; } else { // Must not mix exclusions and inclusions if (exclusions > 0) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java index 3fffbf9d67..1458649bf7 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java @@ -51,16 +51,20 @@ class ProjectionLayer { } public static ProjectionLayer buildLayersForProjection( - Collection dotPaths, List slices, boolean addDocId) { - return buildLayers(dotPaths, slices, true, addDocId); + Collection dotPaths, List slices, boolean addDocId, boolean add$vector) { + return buildLayers(dotPaths, slices, true, addDocId, add$vector); } public static ProjectionLayer buildLayersForIndexing(Collection dotPaths) { - return buildLayers(dotPaths, Collections.emptyList(), false, false); + return buildLayers(dotPaths, Collections.emptyList(), false, false, false); } private static ProjectionLayer buildLayers( - Collection dotPaths, List slices, boolean failOnOverlap, boolean addDocId) { + Collection dotPaths, + List slices, + boolean failOnOverlap, + boolean addDocId, + boolean add$vector) { // Root is always branch (not terminal): ProjectionLayer root = new ProjectionLayer("", false); for (String fullPath : dotPaths) { @@ -83,6 +87,13 @@ private static ProjectionLayer buildLayers( root, new String[] {DocumentConstants.Fields.DOC_ID}); } + if (add$vector) { + buildPath( + failOnOverlap, + DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD, + root, + new String[] {DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD}); + } return root; } From f3f735af7c961d003abd853ab5c4818aebcfc1b2 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 2 Apr 2024 17:44:00 -0700 Subject: [PATCH 04/17] Fix detection of default projection filter --- .../service/projection/DocumentProjector.java | 7 +- .../projection/DocumentProjectorTest.java | 66 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index b659c924ac..b4f501ef5a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -229,8 +229,11 @@ public DocumentProjector buildProjector() { */ boolean isDefaultProjection() { // Only the case if we have no non-doc-id inclusions/exclusions AND - // doc-id is included (by default or explicitly) - return paths.isEmpty() && slices.isEmpty() && !Boolean.FALSE.equals(idInclusion); + // neither doc-id nor $vector has explicit overrides + return paths.isEmpty() + && slices.isEmpty() + && (idInclusion == null) + && ($vectorInclusion == null); } PathCollector collectFromObject(JsonNode ob, String parentPath) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java index a859b182a9..d3ce7051c1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java @@ -753,6 +753,72 @@ void excludeIdIncludeProperty() throws Exception { } """)); } + + @Test + void includeIdExcludeVector() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vector": [0.25, 0.5], + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 1, "$vector": 0 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value": 42 + } + """)); + } + + @Test + void excludeIdIncludeVector() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vector": [0.25, 0.5], + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 0, "$vector": 1 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "$vector": [0.25, 0.5], + "value": 42 + } + """)); + } } // Special case of [data-api#1001]: include-all / exclude-all From f0c63f42fff6b150d9ce65c872b1af5deb0774a3 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 2 Apr 2024 18:47:49 -0700 Subject: [PATCH 05/17] Change default Projection to exclude $vector --- .../service/projection/DocumentProjector.java | 65 +++++++++++-------- .../projection/DocumentProjectorTest.java | 55 ++++++++++++---- 2 files changed, 83 insertions(+), 37 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index b4f501ef5a..372b678df5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.service.projection; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCode; @@ -23,12 +24,6 @@ public class DocumentProjector { * No-op projector that does not modify documents. Considered "exclusion" projector since "no * exclusions" is conceptually what happens ("no inclusions" would drop all content) */ - private static final DocumentProjector DEFAULT_PROJECTOR = - new DocumentProjector(null, false, false); - - private static final DocumentProjector DEFAULT_PROJECTOR_WITH_SIMILARITY = - new DocumentProjector(null, false, true); - private static final DocumentProjector INCLUDE_ALL_PROJECTOR = new DocumentProjector(null, false, false); @@ -57,7 +52,14 @@ private DocumentProjector( } public static DocumentProjector defaultProjector() { - return DEFAULT_PROJECTOR; + return DefaultProjectorWrapper.defaultProjector(); + } + + DocumentProjector withIncludeSimilarity(boolean includeSimilarityScore) { + if (this.includeSimilarityScore == includeSimilarityScore) { + return this; + } + return new DocumentProjector(rootLayer, inclusion, includeSimilarityScore); } public static DocumentProjector createFromDefinition(JsonNode projectionDefinition) { @@ -69,9 +71,9 @@ public static DocumentProjector createFromDefinition( // First special case: "simple" default projection if (projectionDefinition == null || projectionDefinition.isEmpty()) { if (includeSimilarity) { - return DEFAULT_PROJECTOR_WITH_SIMILARITY; + return DefaultProjectorWrapper.defaultProjectorWithSimilarity(); } - return DEFAULT_PROJECTOR; + return DefaultProjectorWrapper.defaultProjector(); } if (!projectionDefinition.isObject()) { throw new JsonApiException( @@ -165,6 +167,34 @@ public int hashCode() { return rootLayer.hashCode(); } + /** + * Due to the way projection is handled, we need to handle construction of default instance via + * separate class (to avoid cyclic dependency) + */ + static class DefaultProjectorWrapper { + /** + * Default projector that drops $vector but otherwise leaves document as-is. Constructed from + * empty definition (no inclusions/exclusions). + */ + private static final DocumentProjector DEFAULT_PROJECTOR; + + static { + ObjectNode emptyDef = new ObjectNode(JsonNodeFactory.instance); + DEFAULT_PROJECTOR = PathCollector.collectPaths(emptyDef, false).buildProjector(); + } + + private static final DocumentProjector DEFAULT_PROJECTOR_WITH_SIMILARITY = + DEFAULT_PROJECTOR.withIncludeSimilarity(true); + + public static DocumentProjector defaultProjector() { + return DEFAULT_PROJECTOR; + } + + public static DocumentProjector defaultProjectorWithSimilarity() { + return DEFAULT_PROJECTOR_WITH_SIMILARITY; + } + } + /** * Helper object used to traverse and collection inclusion/exclusion path definitions and verify * that there are only one or the other (except for doc id). Does not build data structures for @@ -193,10 +223,6 @@ static PathCollector collectPaths(JsonNode def, boolean includeSimilarity) { } public DocumentProjector buildProjector() { - if (isDefaultProjection()) { - return defaultProjector(); - } - // One more thing: do we need to add document id? if (inclusions > 0) { // inclusion-based projection return new DocumentProjector( @@ -223,19 +249,6 @@ public DocumentProjector buildProjector() { } } - /** - * Accessor to use for checking if collected paths indicate "empty" (no-operation) projection: - * if so, caller can avoid actual construction or evaluation. - */ - boolean isDefaultProjection() { - // Only the case if we have no non-doc-id inclusions/exclusions AND - // neither doc-id nor $vector has explicit overrides - return paths.isEmpty() - && slices.isEmpty() - && (idInclusion == null) - && ($vectorInclusion == null); - } - PathCollector collectFromObject(JsonNode ob, String parentPath) { var it = ob.fields(); while (it.hasNext()) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java index d3ce7051c1..995e7056bf 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java @@ -265,25 +265,58 @@ public void verifySliceDefinitionToReturnPositive() throws Exception { } @Nested - class ProjectorApplyInclusions { + class ProjectorApplyDefaultProjection { // [json-api#634]: empty Object same as "include all" @Test - public void testIncludeWithEmptyProject() throws Exception { - final JsonNode doc = - objectMapper.readTree( - """ - { - "_id" : 1, - "value1": 42 - } - """); + public void defaultProjectionRegularFieldsOnly() throws Exception { + final String docJson = + """ + { + "_id" : 1, + "value1": 42 + } + """; DocumentProjector projection = DocumentProjector.createFromDefinition(objectMapper.readTree("{ }")); + final JsonNode doc = objectMapper.readTree(docJson); // Technically considered "Exclusion" but one that excludes nothing assertThat(projection.isInclusion()).isFalse(); projection.applyProjection(doc); - assertThat(doc).isEqualTo(doc); + assertThat(doc).isEqualTo(objectMapper.readTree(docJson)); + } + + @Test + public void defaultProjectionMixAll() throws Exception { + final String docJson = + """ + { + "_id" : 1, + "value1": 42, + "$vector": [0.0, 1.0], + "value2": -3 + } + """; + DocumentProjector projection = + DocumentProjector.createFromDefinition(objectMapper.readTree("{ }")); + final JsonNode doc = objectMapper.readTree(docJson); + + assertThat(projection.isInclusion()).isFalse(); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id" : 1, + "value1": 42, + "value2": -3 + } + """)); } + } + + @Nested + class ProjectorApplyInclusions { @Test public void testSimpleIncludeWithId() throws Exception { From 797d7dfcfe97442464a0106ff3ecc05473a62862 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Wed, 3 Apr 2024 14:55:15 -0700 Subject: [PATCH 06/17] Fix 2 unit tests wrt new default projector --- .../sgv2/jsonapi/service/projection/DocumentProjector.java | 4 ++++ .../service/operation/model/impl/FindOperationTest.java | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index 372b678df5..9b58767e28 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -55,6 +55,10 @@ public static DocumentProjector defaultProjector() { return DefaultProjectorWrapper.defaultProjector(); } + public static DocumentProjector includeAllProjector() { + return INCLUDE_ALL_PROJECTOR; + } + DocumentProjector withIncludeSimilarity(boolean includeSimilarityScore) { if (this.includeSimilarityScore == includeSimilarityScore) { return this; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java index 195920b1db..4a012863bb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java @@ -2643,7 +2643,7 @@ public void vectorSearch() throws Exception { FindOperation.vsearch( VECTOR_COMMAND_CONTEXT, implicitAnd, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null, 2, 2, @@ -2711,7 +2711,7 @@ public void vectorSearchWithFilter() throws Exception { FindOperation.vsearchSingle( VECTOR_COMMAND_CONTEXT, implicitAnd, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, new float[] {0.25f, 0.25f, 0.25f, 0.25f}); From b0f091c64cbc3ca7ba413479a836e6b069ad72bd Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Wed, 3 Apr 2024 15:18:52 -0700 Subject: [PATCH 07/17] Fix more unit tests --- .../model/impl/ReadAndUpdateOperationTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java index f8a5b04834..cb00774250 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java @@ -936,7 +936,7 @@ public void happyPathReplaceUpsert() throws Exception { FindOperation.unsortedSingle( COMMAND_CONTEXT, implicitAnd, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); @@ -971,7 +971,7 @@ public void happyPathReplaceUpsert() throws Exception { false, true, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), 1, 3); @@ -1398,7 +1398,7 @@ public void withUpsert() throws Exception { false, true, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), 1, 3); @@ -1702,7 +1702,7 @@ public void withUpsert() throws Exception { FindOperation.unsorted( COMMAND_CONTEXT, implicitAnd, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null, 21, 20, @@ -1721,7 +1721,7 @@ public void withUpsert() throws Exception { false, true, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), 20, 3); From d19a2b106da10d13b31e50312feecba67b4fb3c6 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Wed, 3 Apr 2024 18:19:10 -0700 Subject: [PATCH 08/17] Improve handling of null docs for update projection --- .../operation/model/impl/ReadAndUpdateOperation.java | 4 +++- .../jsonapi/service/projection/DocumentProjector.java | 1 + .../operation/model/impl/ReadAndUpdateOperationTest.java | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java index 2d4fe8361a..699fd315ad 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperation.java @@ -188,7 +188,9 @@ private Uni processUpdate( returnUpdatedDocument ? updatedDocument : originalDocument; // Some operations (findOneAndUpdate) define projection to apply to // result: - resultProjection.applyProjection(documentToReturn); + if (documentToReturn != null) { // null for some Operation tests + resultProjection.applyProjection(documentToReturn); + } } return new UpdatedDocument( writableShreddedDocument.id(), upsert, documentToReturn, null); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index 9b58767e28..f0666ed76b 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -131,6 +131,7 @@ public void applyProjection(JsonNode document) { } public void applyProjection(JsonNode document, Float similarityScore) { + Objects.requireNonNull(document, "Document to call 'applyProjection()' on must not be null"); // null -> either include-add or exclude-all; but logic may seem counter-intuitive if (rootLayer == null) { if (inclusion) { // exclude-all diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java index cb00774250..8c4c7ecc36 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ReadAndUpdateOperationTest.java @@ -936,7 +936,7 @@ public void happyPathReplaceUpsert() throws Exception { FindOperation.unsortedSingle( COMMAND_CONTEXT, implicitAnd, - DocumentProjector.includeAllProjector(), + DocumentProjector.defaultProjector(), ReadType.DOCUMENT, objectMapper); @@ -971,7 +971,7 @@ public void happyPathReplaceUpsert() throws Exception { false, true, shredder, - DocumentProjector.includeAllProjector(), + DocumentProjector.defaultProjector(), 1, 3); @@ -1398,7 +1398,7 @@ public void withUpsert() throws Exception { false, true, shredder, - DocumentProjector.includeAllProjector(), + DocumentProjector.defaultProjector(), 1, 3); @@ -1702,7 +1702,7 @@ public void withUpsert() throws Exception { FindOperation.unsorted( COMMAND_CONTEXT, implicitAnd, - DocumentProjector.includeAllProjector(), + DocumentProjector.defaultProjector(), null, 21, 20, From 659715f096bdabcd22461360b41988ba08f48088 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Wed, 3 Apr 2024 18:54:04 -0700 Subject: [PATCH 09/17] Fix some of ITs --- ...indOneAndUpdateNoIndexIntegrationTest.java | 4 +- .../api/v1/VectorSearchIntegrationTest.java | 39 ++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java index e1945f0bda..4ee562341c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndUpdateNoIndexIntegrationTest.java @@ -126,8 +126,7 @@ public void byIdAfterUpdate() { "name": "Joe", "age": 42, "enabled": true, - "value": -1, - "$vector" : [ 0.5, -0.25 ] + "value": -1 } """)) .body("status.matchedCount", is(1)) @@ -144,7 +143,6 @@ public void byIdBeforeUpdate() { "name": "Bob", "age": 77, "enabled": true, - "$vector" : [ 0.5, -0.25 ], "value": 3 } """; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index 7f4dc44be5..930b3fdc1f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -202,7 +202,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "1"} + "filter" : {"_id" : "1"}, + "projection": { "$vector": 1 } } } """; @@ -224,8 +225,8 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.documents[0]", jsonEquals(expected)) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected)); } // Test to verify vector embedding size can exceed general Array length limit @@ -243,7 +244,8 @@ public void insertBigVectorThenSearch() { """ { "find": { - "filter" : {"_id" : "bigVector1"} + "filter" : {"_id" : "bigVector1"}, + "projection": { "$vector": 1 } } } """) @@ -455,7 +457,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "2"} + "filter" : {"_id" : "2"}, + "projection": { "$vector": 1 } } } """; @@ -477,8 +480,8 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.documents[0]", jsonEquals(expected)) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.documents[0]", jsonEquals(expected)); } } @@ -1054,6 +1057,7 @@ public void setOperation() { { "findOneAndUpdate": { "filter" : {"_id": "2"}, + "projection": { "$vector": 1 }, "update" : {"$set" : {"$vector" : [0.25, 0.25, 0.25, 0.25, 0.25]}}, "options" : {"returnDocument" : "after"} } @@ -1083,6 +1087,7 @@ public void unsetOperation() { { "findOneAndUpdate": { "filter" : {"name": "Coded Cleats"}, + "projection": { "$vector": 1 }, "update" : {"$unset" : {"$vector" : null}}, "options" : {"returnDocument" : "after"} } @@ -1112,6 +1117,7 @@ public void setOnInsertOperation() { { "findOneAndUpdate": { "filter" : {"_id": "11"}, + "projection": { "$vector": 1 }, "update" : {"$setOnInsert" : {"$vector": [0.11, 0.22, 0.33, 0.44, 0.55]}}, "options" : {"returnDocument" : "after", "upsert": true} } @@ -1176,12 +1182,13 @@ public void setBigVectorOperation() { .contentType(ContentType.JSON) .body( """ - { - "find": { - "filter" : {"_id" : "bigVectorForSet"} - } - } - """) + { + "find": { + "filter" : {"_id" : "bigVectorForSet"}, + "projection": { "$vector": 1 } + } + } + """) .when() .post(CollectionResource.BASE_PATH, namespaceName, bigVectorCollectionName) .then() @@ -1198,6 +1205,7 @@ public void setBigVectorOperation() { { "findOneAndUpdate": { "filter" : {"_id": "bigVectorForSet"}, + "projection": { "$vector": 1 }, "update" : {"$set" : {"$vector" : [ %s ]}}, "options" : {"returnDocument" : "after"} } @@ -1228,7 +1236,8 @@ public void setBigVectorOperation() { """ { "find": { - "filter" : {"_id" : "bigVectorForSet"} + "filter" : {"_id" : "bigVectorForSet"}, + "projection": { "$vector": 1 } } } """) @@ -1332,6 +1341,7 @@ public void findOneAndReplace() { { "findOneAndReplace": { "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, + "projection": { "$vector": 1 }, "replacement" : {"_id" : "3", "username": "user3", "status" : false, "$vector" : [0.12, 0.05, 0.08, 0.32, 0.6]}, "options" : {"returnDocument" : "after"} } @@ -1472,6 +1482,7 @@ public void findOneAndDelete() { """ { "findOneAndDelete": { + "projection": { "$vector": 1 }, "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]} } } From bb52550fd1f5f290c6277ea00334154caeb52b81 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 4 Apr 2024 08:26:51 -0700 Subject: [PATCH 10/17] Fix more ITs (vectorize ones) --- .../v1/VectorizeSearchIntegrationTest.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java index c722a5a96d..83be23bcab 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java @@ -235,7 +235,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "1"} + "filter" : {"_id" : "1"}, + "projection": { "$vector": 1 } } } """; @@ -253,10 +254,10 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("data.documents[0]._id", is("1")) .body("data.documents[0].$vector", is(notNullValue())) - .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)) - .body("errors", is(nullValue())); + .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)); } @Test @@ -460,7 +461,8 @@ public void insertVectorSearch() { """ { "find": { - "filter" : {"_id" : "2"} + "filter" : {"_id" : "2"}, + "projection": { "$vector": 1 } } } """; @@ -473,9 +475,9 @@ public void insertVectorSearch() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("data.documents[0]._id", is("2")) - .body("data.documents[0].$vector", is(notNullValue())) - .body("errors", is(nullValue())); + .body("data.documents[0].$vector", is(notNullValue())); } } @@ -675,6 +677,7 @@ public void vectorizeSortDenyAll() { """ { "find": { + "projection": { "$vector": 1 }, "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"} } } @@ -817,6 +820,7 @@ public void setOperation() { { "findOneAndUpdate": { "filter" : {"_id": "2"}, + "projection": { "$vector": 1 }, "update" : {"$set" : {"description" : "ChatGPT upgraded", "$vectorize" : "ChatGPT upgraded"}}, "options" : {"returnDocument" : "after"} } @@ -881,6 +885,7 @@ public void setOnInsertOperation() { { "findOneAndUpdate": { "filter" : {"_id": "11"}, + "projection": { "$vector": 1 }, "update" : {"$setOnInsert" : {"$vectorize": "New data updated"}}, "options" : {"returnDocument" : "after", "upsert": true} } @@ -1006,6 +1011,7 @@ public void findOneAndReplace() { """ { "findOneAndReplace": { + "projection": { "$vector": 1 }, "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"}, "replacement" : {"_id" : "1", "username": "user1", "status" : false, "description" : "Updating new data", "$vectorize" : "Updating new data"}, "options" : {"returnDocument" : "after"} @@ -1079,7 +1085,8 @@ public void findOneAndDelete() { """ { "findOneAndDelete": { - "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"} + "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"}, + "projection": { "$vector": 1 } } } """; @@ -1097,11 +1104,11 @@ public void findOneAndDelete() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) + .body("status.deletedCount", is(1)) .body("data.document._id", is("1")) .body("data.document.$vector", is(notNullValue())) - .body("data.document.name", is("Coded Cleats")) - .body("status.deletedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document.name", is("Coded Cleats")); } @Test From ac3ce3c8c5ca286065b6e5e979f32320784e52bb Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 4 Apr 2024 08:52:10 -0700 Subject: [PATCH 11/17] Last IT fixes of first round (still a few fails) --- .../api/v1/VectorSearchIntegrationTest.java | 31 +++++++++---------- .../v1/VectorizeSearchIntegrationTest.java | 4 +-- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index 930b3fdc1f..e3c64eef33 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -1072,11 +1072,11 @@ public void setOperation() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document._id", is("2")) - .body("data.document.$vector", contains(0.25f, 0.25f, 0.25f, 0.25f, 0.25f)) + .body("errors", is(nullValue())) .body("status.matchedCount", is(1)) .body("status.modifiedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document._id", is("2")) + .body("data.document.$vector", contains(0.25f, 0.25f, 0.25f, 0.25f, 0.25f)); } @Test @@ -1087,7 +1087,6 @@ public void unsetOperation() { { "findOneAndUpdate": { "filter" : {"name": "Coded Cleats"}, - "projection": { "$vector": 1 }, "update" : {"$unset" : {"$vector" : null}}, "options" : {"returnDocument" : "after"} } @@ -1102,11 +1101,11 @@ public void unsetOperation() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document._id", is("1")) - .body("data.document.$vector", is(nullValue())) + .body("errors", is(nullValue())) .body("status.matchedCount", is(1)) .body("status.modifiedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document._id", is("1")) + .body("data.document.$vector", is(nullValue())); } @Test @@ -1444,12 +1443,12 @@ public void findOneAndReplaceWithBigVector() { .post(CollectionResource.BASE_PATH, namespaceName, bigVectorCollectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("status.matchedCount", is(1)) .body("status.modifiedCount", is(1)) .body("data.document._id", is("bigVectorForFindReplace")) .body("data.document.$vector", is(notNullValue())) - .body("data.document.$vector", hasSize(BIG_VECTOR_SIZE)) - .body("errors", is(nullValue())); + .body("data.document.$vector", hasSize(BIG_VECTOR_SIZE)); // and verify it was set to value with expected size given() @@ -1496,11 +1495,11 @@ public void findOneAndDelete() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) + .body("status.deletedCount", is(1)) .body("data.document._id", is("3")) .body("data.document.$vector", is(notNullValue())) - .body("data.document.name", is("Vision Vector Frame")) - .body("status.deletedCount", is(1)) - .body("errors", is(nullValue())); + .body("data.document.name", is("Vision Vector Frame")); } @Test @@ -1525,9 +1524,9 @@ public void deleteOne() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) + .body("errors", is(nullValue())) .body("status.deletedCount", is(1)) - .body("data", is(nullValue())) - .body("errors", is(nullValue())); + .body("data", is(nullValue())); // ensure find does not find the document json = @@ -1547,9 +1546,9 @@ public void deleteOne() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document", is(nullValue())) + .body("errors", is(nullValue())) .body("status", is(nullValue())) - .body("errors", is(nullValue())); + .body("data.document", is(nullValue())); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java index 83be23bcab..1a86d14538 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java @@ -1107,8 +1107,8 @@ public void findOneAndDelete() { .body("errors", is(nullValue())) .body("status.deletedCount", is(1)) .body("data.document._id", is("1")) - .body("data.document.$vector", is(notNullValue())) - .body("data.document.name", is("Coded Cleats")); + .body("data.document.name", is("Coded Cleats")) + .body("data.document.$vector", is(notNullValue())); } @Test From 43b093492ad5d5334f6f82001aa3920454cdb6ed Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 4 Apr 2024 11:12:28 -0700 Subject: [PATCH 12/17] Fix one more IT --- .../jsonapi/api/v1/VectorSearchIntegrationTest.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index e3c64eef33..f1b3dbce8c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -1408,7 +1408,8 @@ public void findOneAndReplaceWithBigVector() { """ { "find": { - "filter" : {"_id" : "bigVectorForFindReplace"} + "filter" : {"_id" : "bigVectorForFindReplace"}, + "projection": { "*": 1 } } } """) @@ -1428,6 +1429,7 @@ public void findOneAndReplaceWithBigVector() { { "findOneAndReplace": { "filter" : {"_id" : "bigVectorForFindReplace"}, + "projection": { "*": 1 }, "replacement" : {"_id" : "bigVectorForFindReplace", "$vector" : [ %s ]}, "options" : {"returnDocument" : "after"} } @@ -1458,7 +1460,8 @@ public void findOneAndReplaceWithBigVector() { """ { "find": { - "filter" : {"_id" : "bigVectorForFindReplace"} + "filter" : {"_id" : "bigVectorForFindReplace"}, + "projection": { "*": 1 } } } """) @@ -1481,7 +1484,7 @@ public void findOneAndDelete() { """ { "findOneAndDelete": { - "projection": { "$vector": 1 }, + "projection": { "*": 1 }, "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]} } } @@ -1498,8 +1501,8 @@ public void findOneAndDelete() { .body("errors", is(nullValue())) .body("status.deletedCount", is(1)) .body("data.document._id", is("3")) - .body("data.document.$vector", is(notNullValue())) - .body("data.document.name", is("Vision Vector Frame")); + .body("data.document.name", is("Vision Vector Frame")) + .body("data.document.$vector", is(notNullValue())); } @Test From e65cf8b44786d7455a99fdc67ebe1546b2b4e891 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 4 Apr 2024 13:00:14 -0700 Subject: [PATCH 13/17] Use include-all filter for more tests --- .../api/v1/VectorSearchIntegrationTest.java | 18 +++++++++--------- .../api/v1/VectorizeSearchIntegrationTest.java | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index f1b3dbce8c..ca3fe2c5c2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -203,7 +203,7 @@ public void insertVectorSearch() { { "find": { "filter" : {"_id" : "1"}, - "projection": { "$vector": 1 } + "projection": { "*": 1 } } } """; @@ -245,7 +245,7 @@ public void insertBigVectorThenSearch() { { "find": { "filter" : {"_id" : "bigVector1"}, - "projection": { "$vector": 1 } + "projection": { "*": 1 } } } """) @@ -458,7 +458,7 @@ public void insertVectorSearch() { { "find": { "filter" : {"_id" : "2"}, - "projection": { "$vector": 1 } + "projection": { "*": 1 } } } """; @@ -1057,7 +1057,7 @@ public void setOperation() { { "findOneAndUpdate": { "filter" : {"_id": "2"}, - "projection": { "$vector": 1 }, + "projection": { "*": 1 }, "update" : {"$set" : {"$vector" : [0.25, 0.25, 0.25, 0.25, 0.25]}}, "options" : {"returnDocument" : "after"} } @@ -1116,7 +1116,7 @@ public void setOnInsertOperation() { { "findOneAndUpdate": { "filter" : {"_id": "11"}, - "projection": { "$vector": 1 }, + "projection": { "*": 1 }, "update" : {"$setOnInsert" : {"$vector": [0.11, 0.22, 0.33, 0.44, 0.55]}}, "options" : {"returnDocument" : "after", "upsert": true} } @@ -1184,7 +1184,7 @@ public void setBigVectorOperation() { { "find": { "filter" : {"_id" : "bigVectorForSet"}, - "projection": { "$vector": 1 } + "projection": { "*": 1 } } } """) @@ -1204,7 +1204,7 @@ public void setBigVectorOperation() { { "findOneAndUpdate": { "filter" : {"_id": "bigVectorForSet"}, - "projection": { "$vector": 1 }, + "projection": { "*": 1 }, "update" : {"$set" : {"$vector" : [ %s ]}}, "options" : {"returnDocument" : "after"} } @@ -1236,7 +1236,7 @@ public void setBigVectorOperation() { { "find": { "filter" : {"_id" : "bigVectorForSet"}, - "projection": { "$vector": 1 } + "projection": { "*": 1 } } } """) @@ -1340,7 +1340,7 @@ public void findOneAndReplace() { { "findOneAndReplace": { "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, - "projection": { "$vector": 1 }, + "projection": { "*": 1 }, "replacement" : {"_id" : "3", "username": "user3", "status" : false, "$vector" : [0.12, 0.05, 0.08, 0.32, 0.6]}, "options" : {"returnDocument" : "after"} } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java index 1a86d14538..5ad747c5e2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java @@ -1086,7 +1086,7 @@ public void findOneAndDelete() { { "findOneAndDelete": { "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"}, - "projection": { "$vector": 1 } + "projection": { "*": 1 } } } """; From 9b21f44dbe2e972cc05306f3231fc17521e09c41 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 4 Apr 2024 14:13:56 -0700 Subject: [PATCH 14/17] Change defaultProjector() to includeAllProjector() for resolvers, to avoid dropping $vector accidentally --- .../resolver/model/impl/DeleteManyCommandResolver.java | 2 +- .../resolver/model/impl/DeleteOneCommandResolver.java | 6 +++--- .../model/impl/FindOneAndDeleteCommandResolver.java | 6 +++--- .../model/impl/FindOneAndReplaceCommandResolver.java | 6 +++--- .../model/impl/FindOneAndUpdateCommandResolver.java | 10 +++++----- .../resolver/model/impl/UpdateManyCommandResolver.java | 4 ++-- .../resolver/model/impl/UpdateOneCommandResolver.java | 8 ++++---- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java index df49504158..a60d878fec 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteManyCommandResolver.java @@ -59,7 +59,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteMany return FindOperation.unsorted( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null, operationsConfig.maxDocumentDeleteCount() + 1, operationsConfig.defaultPageSize(), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java index 841c9536eb..6931fbbf0e 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/DeleteOneCommandResolver.java @@ -62,7 +62,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteOneC return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.KEY, objectMapper, vector); @@ -74,7 +74,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteOneC return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -88,7 +88,7 @@ private FindOperation getFindOperation(CommandContext commandContext, DeleteOneC return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.KEY, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java index 7d14c686ae..79af76f79c 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndDeleteCommandResolver.java @@ -66,7 +66,7 @@ private FindOperation getFindOperation( return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -78,7 +78,7 @@ private FindOperation getFindOperation( return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -92,7 +92,7 @@ private FindOperation getFindOperation( return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java index 948f1383ee..e808879517 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndReplaceCommandResolver.java @@ -84,7 +84,7 @@ private FindOperation getFindOperation( return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -96,7 +96,7 @@ private FindOperation getFindOperation( return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -110,7 +110,7 @@ private FindOperation getFindOperation( return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java index 34ac8c729c..b9a6056931 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/FindOneAndUpdateCommandResolver.java @@ -86,7 +86,7 @@ private FindOperation getFindOperation( return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -99,8 +99,8 @@ private FindOperation getFindOperation( commandContext, logicalExpression, // 24-Mar-2023, tatu: Since we update the document, need to avoid modifications on - // read path, hence pass identity projector. - DocumentProjector.defaultProjector(), + // read path: + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -115,8 +115,8 @@ private FindOperation getFindOperation( commandContext, logicalExpression, // 24-Mar-2023, tatu: Since we update the document, need to avoid modifications on - // read path, hence pass identity projector. - DocumentProjector.defaultProjector(), + // read path: + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java index 7e2576dfa6..9a6cd05493 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateManyCommandResolver.java @@ -58,7 +58,7 @@ public Operation resolveCommand(CommandContext commandContext, UpdateManyCommand false, upsert, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), operationsConfig.maxDocumentUpdateCount(), operationsConfig.lwt().retries()); } @@ -68,7 +68,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateMany return FindOperation.unsorted( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), null != command.options() ? command.options().pageState() : null, Integer.MAX_VALUE, operationsConfig.defaultPageSize(), diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java index 0fa037501c..ccbbe18e86 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/resolver/model/impl/UpdateOneCommandResolver.java @@ -61,7 +61,7 @@ public Operation resolveCommand(CommandContext commandContext, UpdateOneCommand false, upsert, shredder, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), 1, operationsConfig.lwt().retries()); } @@ -81,7 +81,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateOneC return FindOperation.vsearchSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper, vector); @@ -93,7 +93,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateOneC return FindOperation.sortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), // For in memory sorting we read more data than needed, so defaultSortPageSize like 100 operationsConfig.defaultSortPageSize(), ReadType.SORTED_DOCUMENT, @@ -107,7 +107,7 @@ private FindOperation getFindOperation(CommandContext commandContext, UpdateOneC return FindOperation.unsortedSingle( commandContext, logicalExpression, - DocumentProjector.defaultProjector(), + DocumentProjector.includeAllProjector(), ReadType.DOCUMENT, objectMapper); } From c4422177d4bc7fd0f13ab1071215ce9ed2860bba Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 5 Apr 2024 09:27:16 -0700 Subject: [PATCH 15/17] Add handling for "$vectorize" too --- .../config/constants/DocumentConstants.java | 2 +- .../service/projection/DocumentProjector.java | 26 +++++++++++++------ .../service/projection/ProjectionLayer.java | 20 +++++++++++--- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java b/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java index 3d908260c6..037fbfbe05 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java @@ -31,7 +31,7 @@ interface Fields { String VECTOR_INDEX_FUNCTION_NAME = "similarity_function"; /** Field name used in projection clause to get similarity score in response. */ - String VECTOR_FUNCTION_PROJECTION_FIELD = "$similarity"; + String VECTOR_FUNCTION_SIMILARITY_FIELD = "$similarity"; // Current definition of valid JSON API names: note that this only validates // characters, not length limits (nor empty nor "too long" allowed but validated diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java index f0666ed76b..b1332553cb 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjector.java @@ -140,7 +140,7 @@ public void applyProjection(JsonNode document, Float similarityScore) { // In either case, we may need to add similarity score if present if (includeSimilarityScore && similarityScore != null) { ((ObjectNode) document) - .put(DocumentConstants.Fields.VECTOR_FUNCTION_PROJECTION_FIELD, similarityScore); + .put(DocumentConstants.Fields.VECTOR_FUNCTION_SIMILARITY_FIELD, similarityScore); } return; } @@ -151,7 +151,7 @@ public void applyProjection(JsonNode document, Float similarityScore) { } if (includeSimilarityScore && similarityScore != null) { ((ObjectNode) document) - .put(DocumentConstants.Fields.VECTOR_FUNCTION_PROJECTION_FIELD, similarityScore); + .put(DocumentConstants.Fields.VECTOR_FUNCTION_SIMILARITY_FIELD, similarityScore); } } @@ -178,8 +178,8 @@ public int hashCode() { */ static class DefaultProjectorWrapper { /** - * Default projector that drops $vector but otherwise leaves document as-is. Constructed from - * empty definition (no inclusions/exclusions). + * Default projector that drops $vector and $vectorize fields but otherwise leaves document + * as-is. Constructed from empty definition (no inclusions/exclusions). */ private static final DocumentProjector DEFAULT_PROJECTOR; @@ -212,9 +212,11 @@ private static class PathCollector { private int exclusions, inclusions; - private Boolean idInclusion = null; + private Boolean idInclusion; - private Boolean $vectorInclusion = null; + private Boolean $vectorInclusion; + + private Boolean $vectorizeInclusion; /** Whether similarity score is needed. */ private final boolean includeSimilarityScore; @@ -237,7 +239,9 @@ public DocumentProjector buildProjector() { // doc-id included unless explicitly excluded !Boolean.FALSE.equals(idInclusion), // $vector only included if explicitly included - Boolean.TRUE.equals($vectorInclusion)), + Boolean.TRUE.equals($vectorInclusion), + // $vectorize only included if explicitly included + Boolean.TRUE.equals($vectorizeInclusion)), true, includeSimilarityScore); } else { // exclusion-based @@ -248,7 +252,9 @@ public DocumentProjector buildProjector() { // doc-id excluded only if explicitly excluded Boolean.FALSE.equals(idInclusion), // $vector excluded unless explicitly included - !Boolean.TRUE.equals($vectorInclusion)), + !Boolean.TRUE.equals($vectorInclusion), + // $vectorize excluded unless explicitly included + !Boolean.TRUE.equals($vectorizeInclusion)), false, includeSimilarityScore); } @@ -373,6 +379,8 @@ private void addExclusion(String path) { idInclusion = false; } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(path)) { $vectorInclusion = false; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD.equals(path)) { + $vectorizeInclusion = false; } else { // Must not mix exclusions and inclusions if (inclusions > 0) { @@ -393,6 +401,8 @@ private void addInclusion(String path) { idInclusion = true; } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD.equals(path)) { $vectorInclusion = true; + } else if (DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD.equals(path)) { + $vectorizeInclusion = true; } else { // Must not mix exclusions and inclusions if (exclusions > 0) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java index 1458649bf7..fc62055d90 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/projection/ProjectionLayer.java @@ -51,12 +51,16 @@ class ProjectionLayer { } public static ProjectionLayer buildLayersForProjection( - Collection dotPaths, List slices, boolean addDocId, boolean add$vector) { - return buildLayers(dotPaths, slices, true, addDocId, add$vector); + Collection dotPaths, + List slices, + boolean addDocId, + boolean add$vector, + boolean add$vectorize) { + return buildLayers(dotPaths, slices, true, addDocId, add$vector, add$vectorize); } public static ProjectionLayer buildLayersForIndexing(Collection dotPaths) { - return buildLayers(dotPaths, Collections.emptyList(), false, false, false); + return buildLayers(dotPaths, Collections.emptyList(), false, false, false, false); } private static ProjectionLayer buildLayers( @@ -64,7 +68,8 @@ private static ProjectionLayer buildLayers( List slices, boolean failOnOverlap, boolean addDocId, - boolean add$vector) { + boolean add$vector, + boolean add$vectorize) { // Root is always branch (not terminal): ProjectionLayer root = new ProjectionLayer("", false); for (String fullPath : dotPaths) { @@ -94,6 +99,13 @@ private static ProjectionLayer buildLayers( root, new String[] {DocumentConstants.Fields.VECTOR_EMBEDDING_FIELD}); } + if (add$vectorize) { + buildPath( + failOnOverlap, + DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD, + root, + new String[] {DocumentConstants.Fields.VECTOR_EMBEDDING_TEXT_FIELD}); + } return root; } From 139e4e1642a7a1666bfe88a88c570097ca07ff90 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 5 Apr 2024 10:38:44 -0700 Subject: [PATCH 16/17] Extend unit tests wrt $vectorize --- .../projection/DocumentProjectorTest.java | 95 +++++++++++++++++-- 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java index 995e7056bf..70b2ec23d5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/projection/DocumentProjectorTest.java @@ -292,6 +292,7 @@ public void defaultProjectionMixAll() throws Exception { { "_id" : 1, "value1": 42, + "$vectorize": "Quick brown fox", "$vector": [0.0, 1.0], "value2": -3 } @@ -334,6 +335,7 @@ public void testSimpleIncludeWithId() throws Exception { "nested2" : { "z": 5 }, + "$vectorize": "hello work!", "$vector" : [0.11, 0.22, 0.33, 0.44] } """); @@ -347,7 +349,8 @@ public void testSimpleIncludeWithId() throws Exception { }, "nested.z": 1, "nosuchprop": 1, - "$vector": 1 + "$vector": 1, + "$vectorize": 1 } """)); assertThat(projection.isInclusion()).isTrue(); @@ -356,15 +359,16 @@ public void testSimpleIncludeWithId() throws Exception { .isEqualTo( objectMapper.readTree( """ - { "_id" : 1, - "value2" : false, - "nested" : { - "x": 3, - "z": -1 - }, - "$vector" : [0.11, 0.22, 0.33, 0.44] - } - """)); + { "_id" : 1, + "value2" : false, + "nested" : { + "x": 3, + "z": -1 + }, + "$vectorize": "hello work!", + "$vector" : [0.11, 0.22, 0.33, 0.44] + } + """)); } @Test @@ -383,6 +387,7 @@ public void testSimpleIncludeWithSimilarity() throws Exception { "nested2" : { "z": 5 }, + "$vectorize": "hello work!", "$vector" : [0.11, 0.22, 0.33, 0.44] } """); @@ -398,7 +403,9 @@ public void testSimpleIncludeWithSimilarity() throws Exception { assertThat(projection.isInclusion()).isTrue(); projection.applyProjection(doc, 0.25f); assertThat(doc.get("value2")).isNotNull(); + // we only include $vector explicitly, not $vectorize so: assertThat(doc.get("$vector")).isNotNull(); + assertThat(doc.get("$vectorize")).isNull(); assertThat(doc.get("$similarity").floatValue()).isEqualTo(0.25f); } @@ -852,6 +859,72 @@ void excludeIdIncludeVector() throws Exception { } """)); } + + @Test + void includeIdExcludeVectorize() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vectorize": "Hello world!", + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 1, "$vectorize": 0 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "_id": "id", + "value": 42 + } + """)); + } + + @Test + void excludeIdIncludeVectorize() throws Exception { + final String docJson = + """ + { + "_id": "id", + "$vectorize": "Hello world!", + "value": 42 + } + """; + + // First with filter starting with _id: + DocumentProjector projection = + DocumentProjector.createFromDefinition( + objectMapper.readTree( + """ + { "_id": 0, "$vectorize": 1 } + """)); + // exclusion by default since no regular fields specified + assertThat(projection.isInclusion()).isFalse(); + JsonNode doc = objectMapper.readTree(docJson); + projection.applyProjection(doc); + assertThat(doc) + .isEqualTo( + objectMapper.readTree( + """ + { + "$vectorize": "Hello world!", + "value": 42 + } + """)); + } } // Special case of [data-api#1001]: include-all / exclude-all @@ -864,6 +937,7 @@ public void includeAll() throws Exception { { "_id": "docStarInclude", "value": 42, + "$vectorize": "Quick brown fox", "$vector": [ 1.0, 0.5, -0.25 ] } """; @@ -898,6 +972,7 @@ public void excludeAll() throws Exception { { "_id": "docStarExclude", "value": 42, + "$vectorize": "Quick brown fox", "$vector": [ 1.0, 0.5, -0.25 ] } """; From 5910faa6cb39fa416c83984d5b58284ac561e80a Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 5 Apr 2024 10:49:53 -0700 Subject: [PATCH 17/17] Fix failing IT (wrt exclusion of $vectorize by default) --- .../api/v1/VectorizeSearchIntegrationTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java index 5ad747c5e2..79db453fd9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorizeSearchIntegrationTest.java @@ -665,9 +665,9 @@ public void happyPathWithInvalidData() { .then() .statusCode(200) .body("errors", hasSize(1)) - .body("errors[0].message", startsWith("$vectorize search clause needs to be text value")) .body("errors[0].errorCode", is("SHRED_BAD_VECTORIZE_VALUE")) - .body("errors[0].exceptionClass", is("JsonApiException")); + .body("errors[0].exceptionClass", is("JsonApiException")) + .body("errors[0].message", startsWith("$vectorize search clause needs to be text value")); } @Test @@ -677,7 +677,7 @@ public void vectorizeSortDenyAll() { """ { "find": { - "projection": { "$vector": 1 }, + "projection": { "$vector": 1, "$vectorize" : 1 }, "sort" : {"$vectorize" : "ChatGPT integrated sneakers that talk to you"} } } @@ -699,8 +699,8 @@ public void vectorizeSortDenyAll() { .body("data.documents", hasSize(1)) .body("data.documents[0]._id", is("1")) .body("data.documents[0].$vector", is(notNullValue())) - .body("data.documents[0].$vectorize", is(notNullValue())) - .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)); + .body("data.documents[0].$vector", contains(0.1f, 0.15f, 0.3f, 0.12f, 0.05f)) + .body("data.documents[0].$vectorize", is(notNullValue())); } } @@ -739,8 +739,8 @@ public void happyPath() { .post(CollectionResource.BASE_PATH, namespaceName, collectionName) .then() .statusCode(200) - .body("data.document._id", is("1")) - .body("errors", is(nullValue())); + .body("errors", is(nullValue())) + .body("data.document._id", is("1")); } @Test