From 92e2a7e75d06c4a9c59e1af491db11390485cb7b Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 26 Apr 2023 13:59:40 -0700 Subject: [PATCH 1/4] Bump org.json version for CVE Signed-off-by: Chen Dai --- legacy/build.gradle | 2 +- opensearch/build.gradle | 2 +- ppl/build.gradle | 2 +- prometheus/build.gradle | 2 +- sql/build.gradle | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/legacy/build.gradle b/legacy/build.gradle index d3ee13370e..dd96884346 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -89,7 +89,7 @@ dependencies { } } implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - implementation group: 'org.json', name: 'json', version:'20180813' + implementation group: 'org.json', name: 'json', version:'20230227' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" // add geo module as dependency. https://github.com/opensearch-project/OpenSearch/pull/4180/. diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 6eeab86fff..11f4a9be6b 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" - implementation group: 'org.json', name: 'json', version:'20180813' + implementation group: 'org.json', name: 'json', version:'20230227' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" implementation group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" diff --git a/ppl/build.gradle b/ppl/build.gradle index 5b351b880e..365b8ff0a8 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -48,7 +48,7 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - api group: 'org.json', name: 'json', version: '20180813' + api group: 'org.json', name: 'json', version: '20230227' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' api project(':common') api project(':core') diff --git a/prometheus/build.gradle b/prometheus/build.gradle index b0c05f1bc8..f6b97e66d3 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation 'com.github.babbel:okhttp-aws-signer:1.0.2' implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.1' implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.1' - implementation group: 'org.json', name: 'json', version: '20180813' + implementation group: 'org.json', name: 'json', version: '20230227' testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' diff --git a/sql/build.gradle b/sql/build.gradle index 5c85231b77..0f95b0850f 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -46,7 +46,7 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - implementation group: 'org.json', name: 'json', version:'20180813' + implementation group: 'org.json', name: 'json', version:'20230227' implementation project(':common') implementation project(':core') api project(':protocol') From 7b8aef3a4e00f882bff512a07012a6935061452c Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 26 Apr 2023 14:51:01 -0700 Subject: [PATCH 2/4] Fix assertion by json array similar method Signed-off-by: Chen Dai --- .../org/opensearch/sql/util/MatcherUtils.java | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index cfacefc2c6..e397e1b122 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -231,30 +231,7 @@ public void describeTo(Description description) { @Override protected boolean matchesSafely(JSONArray array) { - if (array.length() != expectedObjects.length) { - return false; - } - - for (int i = 0; i < expectedObjects.length; i++) { - Object expected = expectedObjects[i]; - boolean isEqual; - - // Use similar() because JSONObject/JSONArray.equals() only check if same reference - if (expected instanceof JSONObject) { - isEqual = ((JSONObject) expected).similar(array.get(i)); - } else if (expected instanceof JSONArray) { - isEqual = ((JSONArray) expected).similar(array.get(i)); - } else if (null == expected) { - isEqual = JSONObject.NULL == array.get(i); - } else { - isEqual = expected.equals(array.get(i)); - } - - if (!isEqual) { - return false; - } - } - return true; + return array.similar(new JSONArray(expectedObjects)); } }; } From 073493bdd556e0220e5c2ad01c7c8f86f622a430 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 26 Apr 2023 15:39:24 -0700 Subject: [PATCH 3/4] Fix more assertions which failed on query path return BigDecimal Signed-off-by: Chen Dai --- .../opensearch/sql/legacy/AggregationIT.java | 5 ++-- .../sql/legacy/NestedFieldQueryIT.java | 23 ++++++++++--------- .../org/opensearch/sql/legacy/SubqueryIT.java | 5 +++- .../org/opensearch/sql/util/MatcherUtils.java | 3 ++- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index ba007f43f9..3abf57ddcb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.math.BigDecimal; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -1029,7 +1030,7 @@ public void minOnNestedField() throws Exception { TEST_INDEX_NESTED_TYPE); JSONObject result = executeQuery(query); JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); - Assert.assertEquals(1.0, (double) aggregation.query("/minDays/value"), 0.0001); + Assert.assertEquals(1.0, ((BigDecimal) aggregation.query("/minDays/value")).doubleValue(), 0.0001); } @Test @@ -1039,7 +1040,7 @@ public void sumOnNestedField() throws Exception { TEST_INDEX_NESTED_TYPE); JSONObject result = executeQuery(query); JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); - Assert.assertEquals(19.0, (double) aggregation.query("/sumDays/value"), 0.0001); + Assert.assertEquals(19.0, ((BigDecimal) aggregation.query("/sumDays/value")).doubleValue(), 0.0001); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java index 021f91a81c..8fee963e96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.util.MatcherUtils.kvString; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.function.Function; import org.hamcrest.BaseMatcher; @@ -316,7 +317,7 @@ public void aggregationWithoutGroupBy() throws IOException { JSONObject result = executeQuery(sql); JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); - Assert.assertThat((Double) aggregation.query("/avgDay/value"), closeTo(3.166666666, 0.01)); + Assert.assertThat(((BigDecimal) aggregation.query("/avgDay/value")).doubleValue(), closeTo(3.166666666, 0.01)); } @Test @@ -350,10 +351,10 @@ public void groupByRegularFieldAndSum() throws IOException { Assert.assertNotNull(msgInfoBuckets); Assert.assertThat(msgInfoBuckets.length(), equalTo(2)); Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); - Assert.assertThat((Double) msgInfoBuckets.query("/0/message.dayOfWeek@NESTED/sumDay/value"), + Assert.assertThat(((BigDecimal) msgInfoBuckets.query("/0/message.dayOfWeek@NESTED/sumDay/value")).doubleValue(), closeTo(9.0, 0.01)); Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("b")); - Assert.assertThat((Double) msgInfoBuckets.query("/1/message.dayOfWeek@NESTED/sumDay/value"), + Assert.assertThat(((BigDecimal) msgInfoBuckets.query("/1/message.dayOfWeek@NESTED/sumDay/value")).doubleValue(), closeTo(10.0, 0.01)); } @@ -593,12 +594,12 @@ public void maxAggOnNestedInnerFieldWithoutWhere() throws IOException { Assert.assertThat(bucket.length(), equalTo(2)); Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); Assert.assertThat( - bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max/value"), - equalTo(2015.0)); + ((BigDecimal) bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max/value")).doubleValue(), + closeTo(2015.0, 0.01)); Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); Assert.assertThat( - bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max/value"), - equalTo(2015.0)); + ((BigDecimal) bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max/value")).doubleValue(), + closeTo(2015.0, 0.01)); } @Test @@ -780,12 +781,12 @@ public void havingMaxAggOnNestedInnerFieldWithoutWhere() throws IOException { Assert.assertThat(bucket.length(), equalTo(2)); Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); Assert.assertThat( - bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value"), - equalTo(2015.0)); + ((BigDecimal) bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value")).doubleValue(), + closeTo(2015.0, 0.01)); Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); Assert.assertThat( - bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value"), - equalTo(2015.0)); + ((BigDecimal) bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value")).doubleValue(), + closeTo(2015.0, 0.01)); } /*********************************************************** diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java index 7fbfb1ef1c..0fd0fea7f7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java @@ -7,6 +7,7 @@ package org.opensearch.sql.legacy; import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.Is.is; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; @@ -22,6 +23,7 @@ import com.google.common.collect.Ordering; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -345,7 +347,8 @@ public void selectFromSubqueryCountAndSum() throws IOException { TEST_INDEX_ACCOUNT)); assertThat(result.query("/aggregations/count/value"), equalTo(1000)); - assertThat(result.query("/aggregations/balance/value"), equalTo(25714837.0)); + assertThat(((BigDecimal) result.query("/aggregations/balance/value")).doubleValue(), + closeTo(25714837.0, 0.01)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index e397e1b122..f5fbcf9666 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -21,6 +21,7 @@ import com.google.common.base.Strings; import com.google.gson.JsonParser; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -121,7 +122,7 @@ public static Matcher kvString(String key, Matcher matcher) } public static Matcher kvDouble(String key, Matcher matcher) { - return featureValueOf("Json Match", matcher, actual -> (Double) actual.query(key)); + return featureValueOf("Json Match", matcher, actual -> ((BigDecimal) actual.query(key)).doubleValue()); } public static Matcher kvInt(String key, Matcher matcher) { From cc43408feecf6e6997dc00b455ea033bb545bc8e Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 26 Apr 2023 16:13:34 -0700 Subject: [PATCH 4/4] Fix legacy expr value factory Signed-off-by: Chen Dai --- .../sql/legacy/expression/model/ExprValueFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/expression/model/ExprValueFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/expression/model/ExprValueFactory.java index bc7cb40c31..5dc2b5b50a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/expression/model/ExprValueFactory.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/expression/model/ExprValueFactory.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.expression.model; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -61,6 +62,8 @@ public static ExprValue from(Object o) { return booleanValue((Boolean) o); } else if (o instanceof Double) { return doubleValue((Double) o); + } else if (o instanceof BigDecimal) { + return doubleValue(((BigDecimal) o).doubleValue()); } else if (o instanceof String) { return stringValue((String) o); } else {