From e1f6466ffaf11792e279f1283da01d1000669be0 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Mon, 6 Jun 2022 11:40:12 -0400 Subject: [PATCH 01/11] Version bump 1.3.3 (#413) Signed-off-by: pgodithi --- build.gradle | 4 ++-- jni/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 4291821f9..224e9457b 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.3.2-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "1.3.3-SNAPSHOT") knn_bwc_version = System.getProperty("bwc.version", "1.2.0.0-SNAPSHOT") opensearch_bwc_version = "${knn_bwc_version}" - ".0-SNAPSHOT" opensearch_group = "org.opensearch" @@ -433,7 +433,7 @@ afterEvaluate { requires('opensearch', versions.opensearch, EQUAL) //TODO: Use default knn lib version for now, fix will be compare with version that is used in build step. - requires("opensearch-knnlib", "1.3.2.0", EQUAL) + requires("opensearch-knnlib", "1.3.3.0", EQUAL) packager = 'Amazon' vendor = 'Amazon' os = 'LINUX' diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 7c83375a5..90b26a73d 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -40,7 +40,7 @@ else() endif() if(NOT KNN_PLUGIN_VERSION) - set(KNN_PLUGIN_VERSION "1.3.2.0") + set(KNN_PLUGIN_VERSION "1.3.3.0") endif() # Set architecture specific variables From c058c88814a2cd376e2f9591e095d51f3e579661 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 6 Jun 2022 12:07:05 -0400 Subject: [PATCH 02/11] [Backport 1.3] Change VectorReaderListener to expect number array (#420) Refactors VectorReaderListener onResponse to expect arrays of Number type from search result instead of Double type. Adds test case to confirm that it can handle Integer type. Cleans up tests in VectorReaderTest class. Signed-off-by: John Mazanec (cherry picked from commit 77353512c1f15e0dc996428a982941a7ee3036fb) Signed-off-by: pgodithi --- .../opensearch/knn/training/VectorReader.java | 60 ++-- .../knn/training/VectorReaderTests.java | 286 ++++++++++++------ 2 files changed, 230 insertions(+), 116 deletions(-) diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java index 6392ecabe..9b7db6d99 100644 --- a/src/main/java/org/opensearch/knn/training/VectorReader.java +++ b/src/main/java/org/opensearch/knn/training/VectorReader.java @@ -58,8 +58,15 @@ public VectorReader(Client client) { * @param vectorConsumer consumer used to do something with the collected vectors after each search * @param listener ActionListener that should be called once all search operations complete */ - public void read(ClusterService clusterService, String indexName, String fieldName, int maxVectorCount, - int searchSize, Consumer> vectorConsumer, ActionListener listener) { + public void read( + ClusterService clusterService, + String indexName, + String fieldName, + int maxVectorCount, + int searchSize, + Consumer> vectorConsumer, + ActionListener listener + ) { ValidationException validationException = null; @@ -94,11 +101,17 @@ public void read(ClusterService clusterService, String indexName, String fieldNa // Start reading vectors from index SearchScrollRequestBuilder searchScrollRequestBuilder = createSearchScrollRequestBuilder(); - ActionListener vectorReaderListener = new VectorReaderListener(client, fieldName, - maxVectorCount, 0, listener, vectorConsumer, searchScrollRequestBuilder); - - createSearchRequestBuilder(indexName, fieldName, Integer.min(maxVectorCount, searchSize)) - .execute(vectorReaderListener); + ActionListener vectorReaderListener = new VectorReaderListener( + client, + fieldName, + maxVectorCount, + 0, + listener, + vectorConsumer, + searchScrollRequestBuilder + ); + + createSearchRequestBuilder(indexName, fieldName, Integer.min(maxVectorCount, searchSize)).execute(vectorReaderListener); } private SearchRequestBuilder createSearchRequestBuilder(String indexName, String fieldName, int resultSize) { @@ -142,9 +155,15 @@ private static class VectorReaderListener implements ActionListener listener, Consumer> vectorConsumer, - SearchScrollRequestBuilder searchScrollRequestBuilder) { + public VectorReaderListener( + Client client, + String fieldName, + int maxVectorCount, + int collectedVectorCount, + ActionListener listener, + Consumer> vectorConsumer, + SearchScrollRequestBuilder searchScrollRequestBuilder + ) { this.client = client; this.fieldName = fieldName; this.maxVectorCount = maxVectorCount; @@ -154,7 +173,6 @@ public VectorReaderListener(Client client, String fieldName, int maxVectorCount, this.searchScrollRequestBuilder = searchScrollRequestBuilder; } - @Override @SuppressWarnings("unchecked") public void onResponse(SearchResponse searchResponse) { @@ -165,9 +183,9 @@ public void onResponse(SearchResponse searchResponse) { List trainingData = new ArrayList<>(); for (int i = 0; i < vectorsToAdd; i++) { - trainingData.add(((List) hits[i].getSourceAsMap().get(fieldName)).stream() - .map(Double::floatValue) - .toArray(Float[]::new)); + trainingData.add( + ((List) hits[i].getSourceAsMap().get(fieldName)).stream().map(Number::floatValue).toArray(Float[]::new) + ); } this.collectedVectorCount += trainingData.size(); @@ -180,10 +198,9 @@ public void onResponse(SearchResponse searchResponse) { String scrollId = searchResponse.getScrollId(); if (scrollId != null) { - client.prepareClearScroll().addScrollId(scrollId).execute(ActionListener.wrap( - clearScrollResponse -> listener.onResponse(searchResponse), - listener::onFailure) - ); + client.prepareClearScroll() + .addScrollId(scrollId) + .execute(ActionListener.wrap(clearScrollResponse -> listener.onResponse(searchResponse), listener::onFailure)); } else { listener.onResponse(searchResponse); } @@ -201,10 +218,9 @@ public void onFailure(Exception e) { String scrollId = searchScrollRequestBuilder.request().scrollId(); if (scrollId != null) { - client.prepareClearScroll().addScrollId(scrollId).execute(ActionListener.wrap( - clearScrollResponse -> listener.onFailure(e), - listener::onFailure) - ); + client.prepareClearScroll() + .addScrollId(scrollId) + .execute(ActionListener.wrap(clearScrollResponse -> listener.onFailure(e), listener::onFailure)); } else { listener.onFailure(e); } diff --git a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java index 2b88603d9..0b1e15289 100644 --- a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java +++ b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java @@ -11,9 +11,8 @@ package org.opensearch.knn.training; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; +import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; import org.opensearch.knn.KNNSingleNodeTestCase; @@ -32,32 +31,65 @@ public class VectorReaderTests extends KNNSingleNodeTestCase { - public static Logger logger = LogManager.getLogger(VectorReaderTests.class); + private final static int DEFAULT_LATCH_TIMEOUT = 100; + private final static String DEFAULT_INDEX_NAME = "test-index"; + private final static String DEFAULT_FIELD_NAME = "test-field"; + private final static int DEFAULT_DIMENSION = 16; + private final static int DEFAULT_NUM_VECTORS = 100; + private final static int DEFAULT_MAX_VECTOR_COUNT = 10000; + private final static int DEFAULT_SEARCH_SIZE = 10; public void testRead_valid_completeIndex() throws InterruptedException, ExecutionException, IOException { - // Create an index with knn disabled - String indexName = "test-index"; - String fieldName = "test-field"; - int dim = 16; - int numVectors = 100; - createIndex(indexName); - - // Add a field mapping to the index - createKnnIndexMapping(indexName, fieldName, dim); + createIndex(DEFAULT_INDEX_NAME); + createKnnIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_DIMENSION); // Create list of random vectors and ingest Random random = new Random(); List vectors = new ArrayList<>(); - for (int i = 0; i < numVectors; i++) { - Float[] vector = new Float[dim]; + for (int i = 0; i < DEFAULT_NUM_VECTORS; i++) { + Float[] vector = random.doubles(DEFAULT_DIMENSION).boxed().map(Double::floatValue).toArray(Float[]::new); + vectors.add(vector); + addKnnDoc(DEFAULT_INDEX_NAME, Integer.toString(i), DEFAULT_FIELD_NAME, vector); + } - for (int j = 0; j < dim; j++) { - vector[j] = random.nextFloat(); - } + // Configure VectorReader + ClusterService clusterService = node().injector().getInstance(ClusterService.class); + VectorReader vectorReader = new VectorReader(client()); - vectors.add(vector); + // Read all vectors and confirm they match vectors + TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); + final CountDownLatch inProgressLatch = new CountDownLatch(1); + vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + testVectorConsumer, + createOnSearchResponseCountDownListener(inProgressLatch) + ); + + assertLatchDecremented(inProgressLatch); + + List consumedVectors = testVectorConsumer.getVectorsConsumed(); + assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); + + List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); + List flatConsumedVectors = consumedVectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); + assertEquals(new HashSet<>(flatVectors), new HashSet<>(flatConsumedVectors)); + } - addKnnDoc(indexName, Integer.toString(i), fieldName, vector); + public void testRead_valid_trainVectorsIngestedAsIntegers() throws IOException, ExecutionException, InterruptedException { + createIndex(DEFAULT_INDEX_NAME); + createKnnIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_DIMENSION); + + // Create list of random vectors and ingest + Random random = new Random(); + List vectors = new ArrayList<>(); + for (int i = 0; i < DEFAULT_NUM_VECTORS; i++) { + Integer[] vector = random.ints(DEFAULT_DIMENSION).boxed().toArray(Integer[]::new); + vectors.add(vector); + addKnnDoc(DEFAULT_INDEX_NAME, Integer.toString(i), DEFAULT_FIELD_NAME, vector); } // Configure VectorReader @@ -66,16 +98,23 @@ public void testRead_valid_completeIndex() throws InterruptedException, Executio // Read all vectors and confirm they match vectors TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); - final CountDownLatch inProgressLatch1 = new CountDownLatch(1); - vectorReader.read(clusterService, indexName, fieldName, 10000, 10, testVectorConsumer, - ActionListener.wrap(response -> inProgressLatch1.countDown(), e -> fail(e.toString()))); - - assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); + final CountDownLatch inProgressLatch = new CountDownLatch(1); + vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + testVectorConsumer, + createOnSearchResponseCountDownListener(inProgressLatch) + ); + + assertLatchDecremented(inProgressLatch); List consumedVectors = testVectorConsumer.getVectorsConsumed(); - assertEquals(numVectors, consumedVectors.size()); + assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); - List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); + List flatVectors = vectors.stream().flatMap(Arrays::stream).map(Integer::floatValue).collect(Collectors.toList()); List flatConsumedVectors = consumedVectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); assertEquals(new HashSet<>(flatVectors), new HashSet<>(flatConsumedVectors)); } @@ -83,35 +122,25 @@ public void testRead_valid_completeIndex() throws InterruptedException, Executio public void testRead_valid_incompleteIndex() throws InterruptedException, ExecutionException, IOException { // Check if we get the right number of vectors if the index contains docs that are missing fields // Create an index with knn disabled - String indexName = "test-index"; - String fieldName = "test-field"; - int dim = 16; - int numVectors = 100; - createIndex(indexName); + createIndex(DEFAULT_INDEX_NAME); // Add a field mapping to the index - createKnnIndexMapping(indexName, fieldName, dim); + createKnnIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_DIMENSION); // Create list of random vectors and ingest Random random = new Random(); List vectors = new ArrayList<>(); - for (int i = 0; i < numVectors; i++) { - Float[] vector = new Float[dim]; - - for (int j = 0; j < dim; j++) { - vector[j] = random.nextFloat(); - } - + for (int i = 0; i < DEFAULT_NUM_VECTORS; i++) { + Float[] vector = random.doubles(DEFAULT_DIMENSION).boxed().map(Double::floatValue).toArray(Float[]::new); vectors.add(vector); - - addKnnDoc(indexName, Integer.toString(i ), fieldName, vector); + addKnnDoc(DEFAULT_INDEX_NAME, Integer.toString(i), DEFAULT_FIELD_NAME, vector); } // Create documents that do not have fieldName for training int docsWithoutKNN = 100; String fieldNameWithoutKnn = "test-field-2"; for (int i = 0; i < docsWithoutKNN; i++) { - addDoc(indexName, Integer.toString(i + numVectors), fieldNameWithoutKnn, "dummyValue"); + addDoc(DEFAULT_INDEX_NAME, Integer.toString(i + DEFAULT_NUM_VECTORS), fieldNameWithoutKnn, "dummyValue"); } // Configure VectorReader @@ -120,14 +149,21 @@ public void testRead_valid_incompleteIndex() throws InterruptedException, Execut // Read all vectors and confirm they match vectors TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); - final CountDownLatch inProgressLatch1 = new CountDownLatch(1); - vectorReader.read(clusterService, indexName, fieldName, 10000, 10, testVectorConsumer, - ActionListener.wrap(response -> inProgressLatch1.countDown(), e -> fail(e.toString()))); - - assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); + final CountDownLatch inProgressLatch = new CountDownLatch(1); + vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + testVectorConsumer, + createOnSearchResponseCountDownListener(inProgressLatch) + ); + + assertLatchDecremented(inProgressLatch); List consumedVectors = testVectorConsumer.getVectorsConsumed(); - assertEquals(numVectors, consumedVectors.size()); + assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); List flatConsumedVectors = consumedVectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); @@ -137,26 +173,17 @@ public void testRead_valid_incompleteIndex() throws InterruptedException, Execut public void testRead_valid_OnlyGetMaxVectors() throws InterruptedException, ExecutionException, IOException { // Check if we can limit the number of docs via max operation // Create an index with knn disabled - String indexName = "test-index"; - String fieldName = "test-field"; - int dim = 16; - int numVectorsIndex = 100; int maxNumVectorsRead = 20; - createIndex(indexName); + createIndex(DEFAULT_INDEX_NAME); // Add a field mapping to the index - createKnnIndexMapping(indexName, fieldName, dim); + createKnnIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_DIMENSION); // Create list of random vectors and ingest Random random = new Random(); - for (int i = 0; i < numVectorsIndex; i++) { - Float[] vector = new Float[dim]; - - for (int j = 0; j < dim; j++) { - vector[j] = random.nextFloat(); - } - - addKnnDoc(indexName, Integer.toString(i ), fieldName, vector); + for (int i = 0; i < DEFAULT_NUM_VECTORS; i++) { + Float[] vector = random.doubles(DEFAULT_DIMENSION).boxed().map(Double::floatValue).toArray(Float[]::new); + addKnnDoc(DEFAULT_INDEX_NAME, Integer.toString(i), DEFAULT_FIELD_NAME, vector); } // Configure VectorReader @@ -165,11 +192,18 @@ public void testRead_valid_OnlyGetMaxVectors() throws InterruptedException, Exec // Read maxNumVectorsRead vectors TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); - final CountDownLatch inProgressLatch1 = new CountDownLatch(1); - vectorReader.read(clusterService, indexName, fieldName, maxNumVectorsRead, 10, testVectorConsumer, - ActionListener.wrap(response -> inProgressLatch1.countDown(), e -> fail(e.toString()))); - - assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); + final CountDownLatch inProgressLatch = new CountDownLatch(1); + vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + maxNumVectorsRead, + DEFAULT_SEARCH_SIZE, + testVectorConsumer, + createOnSearchResponseCountDownListener(inProgressLatch) + ); + + assertLatchDecremented(inProgressLatch); List consumedVectors = testVectorConsumer.getVectorsConsumed(); assertEquals(maxNumVectorsRead, consumedVectors.size()); @@ -177,82 +211,138 @@ public void testRead_valid_OnlyGetMaxVectors() throws InterruptedException, Exec public void testRead_invalid_maxVectorCount() { // Create the index - String indexName = "test-index"; - String fieldName = "test-field"; - int dim = 16; - createIndex(indexName); + createIndex(DEFAULT_INDEX_NAME); // Add a field mapping to the index - createKnnIndexMapping(indexName, fieldName, dim); + createKnnIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_DIMENSION); // Configure VectorReader ClusterService clusterService = node().injector().getInstance(ClusterService.class); VectorReader vectorReader = new VectorReader(client()); - expectThrows(ValidationException.class, () -> vectorReader.read(clusterService, indexName, fieldName, -10, 10, null, null)); + int invalidMaxVectorCount = -10; + expectThrows( + ValidationException.class, + () -> vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + invalidMaxVectorCount, + DEFAULT_SEARCH_SIZE, + null, + null + ) + ); } public void testRead_invalid_searchSize() { // Create the index - String indexName = "test-index"; - String fieldName = "test-field"; - int dim = 16; - createIndex(indexName); + createIndex(DEFAULT_INDEX_NAME); // Add a field mapping to the index - createKnnIndexMapping(indexName, fieldName, dim); + createKnnIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_DIMENSION); // Configure VectorReader ClusterService clusterService = node().injector().getInstance(ClusterService.class); VectorReader vectorReader = new VectorReader(client()); // Search size is negative - expectThrows(ValidationException.class, () -> vectorReader.read(clusterService, indexName, fieldName, 100, -10, null, null)); + int invalidSearchSize1 = -10; + expectThrows( + ValidationException.class, + () -> vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + invalidSearchSize1, + null, + null + ) + ); // Search size is greater than 10000 - expectThrows(ValidationException.class, () -> vectorReader.read(clusterService, indexName, fieldName, 100, 20000, null, null)); + int invalidSearchSize2 = 20000; + expectThrows( + ValidationException.class, + () -> vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + invalidSearchSize2, + null, + null + ) + ); } public void testRead_invalid_indexDoesNotExist() { // Check that read throws a validation exception when the index does not exist - String indexName = "test-index"; - String fieldName = "test-field"; - // Configure VectorReader ClusterService clusterService = node().injector().getInstance(ClusterService.class); VectorReader vectorReader = new VectorReader(client()); // Should throw a validation exception because index does not exist - expectThrows(ValidationException.class, () -> vectorReader.read(clusterService, indexName, fieldName, 10000, 10, null, null)); + expectThrows( + ValidationException.class, + () -> vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + null, + null + ) + ); } public void testRead_invalid_fieldDoesNotExist() { // Check that read throws a validation exception when the field does not exist - String indexName = "test-index"; - String fieldName = "test-field"; - createIndex(indexName); + createIndex(DEFAULT_INDEX_NAME); // Configure VectorReader ClusterService clusterService = node().injector().getInstance(ClusterService.class); VectorReader vectorReader = new VectorReader(client()); // Should throw a validation exception because field is not k-NN - expectThrows(ValidationException.class, () -> vectorReader.read(clusterService, indexName, fieldName, 10000, 10, null, null)); + expectThrows( + ValidationException.class, + () -> vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + null, + null + ) + ); } public void testRead_invalid_fieldIsNotKnn() throws InterruptedException, ExecutionException, IOException { // Check that read throws a validation exception when the field does not exist - String indexName = "test-index"; - String fieldName = "test-field"; - createIndex(indexName); - addDoc(indexName, "test-id", fieldName, "dummy"); + createIndex(DEFAULT_INDEX_NAME); + addDoc(DEFAULT_INDEX_NAME, "test-id", DEFAULT_FIELD_NAME, "dummy"); // Configure VectorReader ClusterService clusterService = node().injector().getInstance(ClusterService.class); VectorReader vectorReader = new VectorReader(client()); // Should throw a validation exception because field does not exist - expectThrows(ValidationException.class, () -> vectorReader.read(clusterService, indexName, fieldName, 10000, 10, null, null)); + expectThrows( + ValidationException.class, + () -> vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_FIELD_NAME, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + null, + null + ) + ); } private static class TestVectorConsumer implements Consumer> { @@ -272,4 +362,12 @@ public List getVectorsConsumed() { return vectorsConsumed; } } -} \ No newline at end of file + + private void assertLatchDecremented(CountDownLatch countDownLatch) throws InterruptedException { + assertTrue(countDownLatch.await(DEFAULT_LATCH_TIMEOUT, TimeUnit.SECONDS)); + } + + private ActionListener createOnSearchResponseCountDownListener(CountDownLatch countDownLatch) { + return ActionListener.wrap(response -> countDownLatch.countDown(), Throwable::printStackTrace); + } +} From b8a3a5ddc9fda92c1578154936ff77b00afd30db Mon Sep 17 00:00:00 2001 From: pgodithi Date: Tue, 5 Jul 2022 07:39:37 -0400 Subject: [PATCH 03/11] Staging version increment automation Signed-off-by: pgodithi --- build.gradle | 26 ++++++++++++++++++++++++-- gradle.properties | 11 ++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 224e9457b..d279baf71 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.3.3-SNAPSHOT") + opensearch_version = project.property('opensearch.version') knn_bwc_version = System.getProperty("bwc.version", "1.2.0.0-SNAPSHOT") opensearch_bwc_version = "${knn_bwc_version}" - ".0-SNAPSHOT" opensearch_group = "org.opensearch" @@ -63,7 +63,7 @@ if (!usingRemoteCluster) { } ext { - isSnapshot = "true" == System.getProperty("build.snapshot", "true") + isSnapshot = project.property('build.snapshot') projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.TXT') noticeFile = rootProject.file('NOTICE.TXT') @@ -225,6 +225,28 @@ task integTestRemote(type: RestIntegTestTask) { } } +task versionIncrement { + final String workingDir = project.buildDir.toString() + "/../" + if(project.hasProperty('newVersion')) { + println 'Set Project to new Version '+newVersion.tokenize('-')[0] + ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { + fileset(dir: workingDir) { + include(name: "jni/CMakeLists.txt") + include(name: "build.gradle") + } + } + } +} + +task setVersion(dependsOn: versionIncrement) { + if(project.hasProperty('newVersion')) { + ant.propertyfile( + file: "gradle.properties") { + entry( key: "opensearch.version", value: "${newVersion}") + } + } +} + // bwcFilePath contains the gradlew assemble binary files of k-NN plugins String baseName = "knnBwcCluster" String bwcFilePath = "src/test/resources/org/opensearch/knn/bwc/" diff --git a/gradle.properties b/gradle.properties index 8c7ff252a..e53cf569d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,7 @@ -# -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# +#Tue, 05 Jul 2022 07:37:17 -0400 -version=1.0.0 +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +opensearch.version=1.3.3-SNAPSHOT +build.snapshot=true From 70eb0111d2b9d1b5fdf0254fd54f68512807c538 Mon Sep 17 00:00:00 2001 From: opensearch-ci-bot Date: Tue, 5 Jul 2022 11:42:40 +0000 Subject: [PATCH 04/11] Version Increment. Signed-off-by: opensearch-ci-bot Signed-off-by: pgodithi --- build.gradle | 2 +- gradle.properties | 4 ++-- jni/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index d279baf71..f7d4253ec 100644 --- a/build.gradle +++ b/build.gradle @@ -455,7 +455,7 @@ afterEvaluate { requires('opensearch', versions.opensearch, EQUAL) //TODO: Use default knn lib version for now, fix will be compare with version that is used in build step. - requires("opensearch-knnlib", "1.3.3.0", EQUAL) + requires("opensearch-knnlib", "1.3.4.0", EQUAL) packager = 'Amazon' vendor = 'Amazon' os = 'LINUX' diff --git a/gradle.properties b/gradle.properties index e53cf569d..1ec41a76a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ -#Tue, 05 Jul 2022 07:37:17 -0400 +#Tue, 05 Jul 2022 11:42:39 +0000 # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 -opensearch.version=1.3.3-SNAPSHOT +opensearch.version=1.3.4-SNAPSHOT build.snapshot=true diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 90b26a73d..d1e72bd2b 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -40,7 +40,7 @@ else() endif() if(NOT KNN_PLUGIN_VERSION) - set(KNN_PLUGIN_VERSION "1.3.3.0") + set(KNN_PLUGIN_VERSION "1.3.4.0") endif() # Set architecture specific variables From b730e991fffc253c9734acecc16f8af67e79201f Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 5 Jul 2022 07:46:03 -0400 Subject: [PATCH 05/11] Revert "[AUTO] Version Increment to 1.3.4-SNAPSHOT" Signed-off-by: pgodithi --- build.gradle | 2 +- gradle.properties | 4 ++-- jni/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index f7d4253ec..d279baf71 100644 --- a/build.gradle +++ b/build.gradle @@ -455,7 +455,7 @@ afterEvaluate { requires('opensearch', versions.opensearch, EQUAL) //TODO: Use default knn lib version for now, fix will be compare with version that is used in build step. - requires("opensearch-knnlib", "1.3.4.0", EQUAL) + requires("opensearch-knnlib", "1.3.3.0", EQUAL) packager = 'Amazon' vendor = 'Amazon' os = 'LINUX' diff --git a/gradle.properties b/gradle.properties index 1ec41a76a..e53cf569d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ -#Tue, 05 Jul 2022 11:42:39 +0000 +#Tue, 05 Jul 2022 07:37:17 -0400 # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 -opensearch.version=1.3.4-SNAPSHOT +opensearch.version=1.3.3-SNAPSHOT build.snapshot=true diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index d1e72bd2b..90b26a73d 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -40,7 +40,7 @@ else() endif() if(NOT KNN_PLUGIN_VERSION) - set(KNN_PLUGIN_VERSION "1.3.4.0") + set(KNN_PLUGIN_VERSION "1.3.3.0") endif() # Set architecture specific variables From 4a7e7027687c8c720f94b065d534dd01bce13846 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Tue, 5 Jul 2022 07:49:33 -0400 Subject: [PATCH 06/11] Staging version increment automation Signed-off-by: pgodithi --- gradle.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/gradle.properties b/gradle.properties index e53cf569d..b534871d6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,5 +3,6 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 +version=1.0.0 opensearch.version=1.3.3-SNAPSHOT build.snapshot=true From ebf2c1f933e230fd2cd839a25cc41480512ff38e Mon Sep 17 00:00:00 2001 From: pgodithi Date: Tue, 5 Jul 2022 07:51:50 -0400 Subject: [PATCH 07/11] Version increment automation Signed-off-by: pgodithi --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b534871d6..178d3d0a2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,6 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 -version=1.0.0 opensearch.version=1.3.3-SNAPSHOT build.snapshot=true +version=1.0.0 From fa1092c37dd084b7007960d4fd1dbb567b4dae86 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 8 Jul 2022 13:11:25 -0400 Subject: [PATCH 08/11] Version increment automation Signed-off-by: pgodithi --- build.gradle | 41 +++++++++++++++++------------------------ gradle.properties | 2 -- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/build.gradle b/build.gradle index d279baf71..2f0a94fd2 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = project.property('opensearch.version') + opensearch_version = System.getProperty("opensearch.version", "1.3.3-SNAPSHOT") knn_bwc_version = System.getProperty("bwc.version", "1.2.0.0-SNAPSHOT") opensearch_bwc_version = "${knn_bwc_version}" - ".0-SNAPSHOT" opensearch_group = "org.opensearch" @@ -63,7 +63,7 @@ if (!usingRemoteCluster) { } ext { - isSnapshot = project.property('build.snapshot') + isSnapshot = "true" == System.getProperty("build.snapshot", "true") projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.TXT') noticeFile = rootProject.file('NOTICE.TXT') @@ -225,28 +225,6 @@ task integTestRemote(type: RestIntegTestTask) { } } -task versionIncrement { - final String workingDir = project.buildDir.toString() + "/../" - if(project.hasProperty('newVersion')) { - println 'Set Project to new Version '+newVersion.tokenize('-')[0] - ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { - fileset(dir: workingDir) { - include(name: "jni/CMakeLists.txt") - include(name: "build.gradle") - } - } - } -} - -task setVersion(dependsOn: versionIncrement) { - if(project.hasProperty('newVersion')) { - ant.propertyfile( - file: "gradle.properties") { - entry( key: "opensearch.version", value: "${newVersion}") - } - } -} - // bwcFilePath contains the gradlew assemble binary files of k-NN plugins String baseName = "knnBwcCluster" String bwcFilePath = "src/test/resources/org/opensearch/knn/bwc/" @@ -498,3 +476,18 @@ afterEvaluate { tasks = ['build', 'buildRpm', 'buildDeb'] } } + +task versionIncrement { + onlyIf { System.getProperty('newVersion') } + doLast { + ext.newVersion = System.getProperty('newVersion') + println "Setting version to ${newVersion}." + final String workingDir = project.buildDir.toString() + "/../" + ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { + fileset(dir: workingDir) { + include(name: "jni/CMakeLists.txt") + include(name: "build.gradle") + } + } + } +} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 178d3d0a2..1aebc15b5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,4 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 -opensearch.version=1.3.3-SNAPSHOT -build.snapshot=true version=1.0.0 From 3759577e32d36614489a3267430baf1022946637 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 8 Jul 2022 13:12:33 -0400 Subject: [PATCH 09/11] Version increment automation Signed-off-by: pgodithi --- gradle.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index 1aebc15b5..8c7ff252a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ -#Tue, 05 Jul 2022 07:37:17 -0400 - -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# version=1.0.0 From 9989d2ea89593192a387d452f86212e7a1fd754d Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 8 Jul 2022 14:09:28 -0400 Subject: [PATCH 10/11] Version increment automation Signed-off-by: pgodithi --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 2f0a94fd2..79eca6bb0 100644 --- a/build.gradle +++ b/build.gradle @@ -482,9 +482,9 @@ task versionIncrement { doLast { ext.newVersion = System.getProperty('newVersion') println "Setting version to ${newVersion}." - final String workingDir = project.buildDir.toString() + "/../" + // String tokenization to support -SNAPSHOT ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { - fileset(dir: workingDir) { + fileset(dir: projectDir) { include(name: "jni/CMakeLists.txt") include(name: "build.gradle") } From fbfaed864848b38e42245bf09fc5255f3b803011 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 8 Jul 2022 14:18:57 -0400 Subject: [PATCH 11/11] Version increment automation Signed-off-by: pgodithi --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index 79eca6bb0..e49a54904 100644 --- a/build.gradle +++ b/build.gradle @@ -477,6 +477,7 @@ afterEvaluate { } } +// versionIncrement: Task to auto increment to the next development iteration task versionIncrement { onlyIf { System.getProperty('newVersion') } doLast { @@ -485,6 +486,7 @@ task versionIncrement { // String tokenization to support -SNAPSHOT ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { fileset(dir: projectDir) { + // Include the required files that needs to be updated with new Version include(name: "jni/CMakeLists.txt") include(name: "build.gradle") }