From 26df33132b9edfffb68458d77d218074d9255f8c Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Wed, 30 Aug 2023 08:51:37 -0700 Subject: [PATCH 1/5] fix NPE when concurrent delete --- .../service/operation/model/impl/DeleteOperation.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DeleteOperation.java b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DeleteOperation.java index ad1d6786db..09e970b90a 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DeleteOperation.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/DeleteOperation.java @@ -127,7 +127,10 @@ public Uni> execute(QueryExecutor queryExecutor) { Tuple3.of( deleted != null ? deleted.getItem1() : false, error, - error == null && returnDocumentInResponse + error == null + && deleted != null + && deleted.getItem2() != null + && returnDocumentInResponse ? applyProjection(deleted.getItem2()) : document)); }) From 851f06621d096d6f0d63d0bfb7c03bdf29f0a6f5 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 31 Aug 2023 09:59:45 -0700 Subject: [PATCH 2/5] change IT cassandra image-tag, add concurrent delete IT --- pom.xml | 4 +- .../v1/FindOneAndDeleteIntegrationTest.java | 161 +++++++++++++++++- .../api/v1/VectorSearchIntegrationTest.java | 122 ++++++++++++- 3 files changed, 267 insertions(+), 20 deletions(-) diff --git a/pom.xml b/pom.xml index 2ee44bca79..dba846898e 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ stargateio/dse-next - 4.0.7-336cdd7405ee + 4.0.7-e47eb8e14b96 stargateio/coordinator-dse-next v${stargate.version} dse-next-${stargate.int-test.cassandra.image-tag}-cluster @@ -155,7 +155,7 @@ verify - + xx ${project.build.directory}/${project.build.finalName}-runner org.jboss.logmanager.LogManager ${stargate.int-test.cassandra.image}:${stargate.int-test.cassandra.image-tag} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndDeleteIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndDeleteIntegrationTest.java index 9f78cb82d7..caf69d0d7a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndDeleteIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindOneAndDeleteIntegrationTest.java @@ -3,21 +3,17 @@ import static io.restassured.RestAssured.given; import static io.stargate.sgv2.common.IntegrationTestUtils.getAuthToken; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.*; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.restassured.http.ContentType; import io.stargate.sgv2.api.common.config.constants.HttpConstants; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.ClassOrderer; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestClassOrder; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReferenceArray; +import org.junit.jupiter.api.*; @QuarkusIntegrationTest @QuarkusTestResource(DseTestResource.class) @@ -321,4 +317,151 @@ public void checkMetrics() { FindOneAndDeleteIntegrationTest.super.checkMetrics("FindOneAndDeleteCommand"); } } + + @Nested + class ConcurrentDelete { + + @Test + public void findOneAndDelete() throws Exception { + insertDocuments(); + String json = + """ + { + "findOneAndDelete": { + "filter" : {"name" : "Logic Layers"} + } + } + """; + + int threads = 5; + AtomicReferenceArray assertionErrors = new AtomicReferenceArray<>(threads); + CountDownLatch latch = new CountDownLatch(threads); + + for (int i = 0; i < threads; i++) { + int index = i; + Thread thread = + new Thread( + () -> { + try { + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.deletedCount", anyOf(is(0), is(1))) + .body("errors", is(nullValue())); + } catch (AssertionError e) { + assertionErrors.set(index, e); + } finally { + latch.countDown(); + } + }); + thread.start(); + } + latch.await(); + int assertionErrorCount = 0; + for (int i = 0; i < threads; i++) { + AssertionError assertionError = assertionErrors.get(i); + if (null != assertionError) { + assertionErrorCount++; + } + } + assertThat(assertionErrorCount).isEqualTo(0); + } + + @Test + public void findOneAndDeleteProjection() throws Exception { + insertDocuments(); + String json = + """ + { + "findOneAndDelete": { + "filter" : {"name" : "Coded Cleats"}, + "projection" : {"name" : 1} + } + } + """; + int threads = 5; + AtomicReferenceArray assertionErrors = new AtomicReferenceArray<>(threads); + CountDownLatch latch = new CountDownLatch(threads); + + for (int i = 0; i < threads; i++) { + int index = i; + Thread thread = + new Thread( + () -> { + try { + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.deletedCount", anyOf(is(0), is(1))) + .body("errors", is(nullValue())); + } catch (AssertionError e) { + assertionErrors.set(index, e); + } finally { + latch.countDown(); + } + }); + thread.start(); + } + latch.await(); + int assertionErrorCount = 0; + for (int i = 0; i < threads; i++) { + AssertionError assertionError = assertionErrors.get(i); + if (null != assertionError) { + assertionErrorCount++; + } + } + assertThat(assertionErrorCount).isEqualTo(0); + } + } + + public void insertDocuments() { + String json = + """ + { + "insertMany": { + "documents": [ + { + "_id": "1", + "name": "Coded Cleats", + "description": "ChatGPT integrated sneakers that talk to you" + }, + { + "_id": "2", + "name": "Logic Layers", + "description": "An AI quilt to help you sleep forever" + }, + { + "_id": "3", + "name": "Logic Layers", + "description": "An AI quilt to help you sleep forever" + }, + { + "_id": "4", + "name": "Vision Vector Frame", + "description": "Vision Vector Frame', 'A deep learning display that controls your mood" + } + ] + } + } + """; + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .body("status.insertedIds[0]", not(emptyString())) + .statusCode(200); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index 02e6cd252d..26ae8c0b90 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -3,15 +3,9 @@ import static io.restassured.RestAssured.given; import static io.stargate.sgv2.common.IntegrationTestUtils.getAuthToken; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anyOf; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; @@ -19,6 +13,8 @@ import io.stargate.sgv2.api.common.config.constants.HttpConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCode; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReferenceArray; import org.junit.jupiter.api.ClassOrderer; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Nested; @@ -1683,6 +1679,114 @@ public void findOneAndReplace() { } } + @Nested + @Order(8) + @TestMethodOrder(MethodOrderer.OrderAnnotation.class) + class ConcurrentDelete { + + @Test + public void findOneAndDelete() throws Exception { + insertVectorDocuments(); + String json = + """ + { + "findOneAndDelete": { + "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]} + } + } + """; + + int threads = 5; + AtomicReferenceArray assertionErrors = new AtomicReferenceArray<>(threads); + CountDownLatch latch = new CountDownLatch(threads); + + for (int i = 0; i < threads; i++) { + int index = i; + Thread thread = + new Thread( + () -> { + try { + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.deletedCount", anyOf(is(0), is(1))) + .body("errors", is(nullValue())); + } catch (AssertionError e) { + assertionErrors.set(index, e); + } finally { + latch.countDown(); + } + }); + thread.start(); + } + latch.await(); + int assertionErrorCount = 0; + for (int i = 0; i < threads; i++) { + AssertionError assertionError = assertionErrors.get(i); + if (null != assertionError) { + assertionErrorCount++; + } + } + assertThat(assertionErrorCount).isEqualTo(0); + } + + @Test + public void findOneAndDeleteProjection() throws Exception { + insertVectorDocuments(); + String json = + """ + { + "findOneAndDelete": { + "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, + "projection" : {"name" : 1} + } + } + """; + int threads = 5; + AtomicReferenceArray assertionErrors = new AtomicReferenceArray<>(threads); + CountDownLatch latch = new CountDownLatch(threads); + + for (int i = 0; i < threads; i++) { + int index = i; + Thread thread = + new Thread( + () -> { + try { + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, namespaceName, collectionName) + .then() + .statusCode(200) + .body("status.deletedCount", anyOf(is(0), is(1))) + .body("errors", is(nullValue())); + } catch (AssertionError e) { + assertionErrors.set(index, e); + } finally { + latch.countDown(); + } + }); + thread.start(); + } + latch.await(); + int assertionErrorCount = 0; + for (int i = 0; i < threads; i++) { + AssertionError assertionError = assertionErrors.get(i); + if (null != assertionError) { + assertionErrorCount++; + } + } + assertThat(assertionErrorCount).isEqualTo(0); + } + } + private static void createVectorCollection( String namespaceName, String collectionName, int vectorSize) { given() From 79b73d4e6e2fb76e664a61de3a40b22dd163991d Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 31 Aug 2023 10:04:55 -0700 Subject: [PATCH 3/5] typo --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dba846898e..4623a7b522 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ verify - xx + ${project.build.directory}/${project.build.finalName}-runner org.jboss.logmanager.LogManager ${stargate.int-test.cassandra.image}:${stargate.int-test.cassandra.image-tag} From 7406863046bb17dcd316d09d7c326cf49dc70d27 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 31 Aug 2023 15:50:37 -0700 Subject: [PATCH 4/5] delete duplicate codes --- .../api/v1/VectorSearchIntegrationTest.java | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index 26ae8c0b90..f88efc0216 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -1678,115 +1678,6 @@ public void findOneAndReplace() { is(ErrorCode.VECTOR_SEARCH_SIMILARITY_PROJECTION_NOT_SUPPORTED.getMessage())); } } - - @Nested - @Order(8) - @TestMethodOrder(MethodOrderer.OrderAnnotation.class) - class ConcurrentDelete { - - @Test - public void findOneAndDelete() throws Exception { - insertVectorDocuments(); - String json = - """ - { - "findOneAndDelete": { - "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]} - } - } - """; - - int threads = 5; - AtomicReferenceArray assertionErrors = new AtomicReferenceArray<>(threads); - CountDownLatch latch = new CountDownLatch(threads); - - for (int i = 0; i < threads; i++) { - int index = i; - Thread thread = - new Thread( - () -> { - try { - given() - .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) - .contentType(ContentType.JSON) - .body(json) - .when() - .post(CollectionResource.BASE_PATH, namespaceName, collectionName) - .then() - .statusCode(200) - .body("status.deletedCount", anyOf(is(0), is(1))) - .body("errors", is(nullValue())); - } catch (AssertionError e) { - assertionErrors.set(index, e); - } finally { - latch.countDown(); - } - }); - thread.start(); - } - latch.await(); - int assertionErrorCount = 0; - for (int i = 0; i < threads; i++) { - AssertionError assertionError = assertionErrors.get(i); - if (null != assertionError) { - assertionErrorCount++; - } - } - assertThat(assertionErrorCount).isEqualTo(0); - } - - @Test - public void findOneAndDeleteProjection() throws Exception { - insertVectorDocuments(); - String json = - """ - { - "findOneAndDelete": { - "sort" : {"$vector" : [0.15, 0.1, 0.1, 0.35, 0.55]}, - "projection" : {"name" : 1} - } - } - """; - int threads = 5; - AtomicReferenceArray assertionErrors = new AtomicReferenceArray<>(threads); - CountDownLatch latch = new CountDownLatch(threads); - - for (int i = 0; i < threads; i++) { - int index = i; - Thread thread = - new Thread( - () -> { - try { - given() - .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) - .contentType(ContentType.JSON) - .body(json) - .when() - .post(CollectionResource.BASE_PATH, namespaceName, collectionName) - .then() - .statusCode(200) - .body("status.deletedCount", anyOf(is(0), is(1))) - .body("errors", is(nullValue())); - } catch (AssertionError e) { - assertionErrors.set(index, e); - } finally { - latch.countDown(); - } - }); - thread.start(); - } - latch.await(); - int assertionErrorCount = 0; - for (int i = 0; i < threads; i++) { - AssertionError assertionError = assertionErrors.get(i); - if (null != assertionError) { - assertionErrorCount++; - } - } - assertThat(assertionErrorCount).isEqualTo(0); - } - } - private static void createVectorCollection( String namespaceName, String collectionName, int vectorSize) { given() From 575882003e1f70f4bcd745f93ed3e8aeac674c54 Mon Sep 17 00:00:00 2001 From: Yuqi Du Date: Thu, 31 Aug 2023 15:53:52 -0700 Subject: [PATCH 5/5] delete duplicate codes + format --- .../sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java index f88efc0216..053360d91b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/VectorSearchIntegrationTest.java @@ -3,9 +3,7 @@ import static io.restassured.RestAssured.given; import static io.stargate.sgv2.common.IntegrationTestUtils.getAuthToken; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; -import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.*; -import static org.hamcrest.Matchers.anyOf; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; @@ -13,8 +11,6 @@ import io.stargate.sgv2.api.common.config.constants.HttpConstants; import io.stargate.sgv2.jsonapi.exception.ErrorCode; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicReferenceArray; import org.junit.jupiter.api.ClassOrderer; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Nested; @@ -1678,6 +1674,7 @@ public void findOneAndReplace() { is(ErrorCode.VECTOR_SEARCH_SIMILARITY_PROJECTION_NOT_SUPPORTED.getMessage())); } } + private static void createVectorCollection( String namespaceName, String collectionName, int vectorSize) { given()