Skip to content

Commit

Permalink
[Tests] IT tests and test utils update to fix failing tests for serve…
Browse files Browse the repository at this point in the history 
…rless (#2902)

Signed-off-by: Manasvini B S <manasvis@amazon.com>
  • Loading branch information
@manasvinibs
manasvinibs committed Aug 13, 2024
1 parent 4a735ea commit 6972487
Show file tree
Hide file tree
Showing 11 changed files with 491 additions and 91 deletions.
29 changes: 22 additions & 7 deletions integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -689,13 +690,18 @@ public void sanitizeTest() throws IOException {
String.format(
Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE),
false);
List<String> lines = csvResult.getLines();
assertEquals(5, lines.size());
assertEquals(lines.get(0), "'+Amber JOHnny,Duke Willmington+");
assertEquals(lines.get(1), "'-Hattie,Bond-");
assertEquals(lines.get(2), "'=Nanette,Bates=");
assertEquals(lines.get(3), "'@Dale,Adams@");
assertEquals(lines.get(4), "\",Elinor\",\"Ratliff,,,\"");
List<String> actualLines = csvResult.getLines();
assertEquals(5, actualLines.size());

List<String> expectedLines =
Arrays.asList(
"'+Amber JOHnny,Duke Willmington+",
"'-Hattie,Bond-",
"'=Nanette,Bates=",
"'@Dale,Adams@",
"\",Elinor\",\"Ratliff,,,\"");

assertContainsSameItems(expectedLines, actualLines);
}

@Test
Expand All @@ -719,6 +725,15 @@ private void verifyFieldOrder(final String[] expectedFields) throws IOException
verifyFieldOrder(expectedFields, query);
}

private void assertContainsSameItems(List<String> expectedLines, List<String> actualLines) {
Collections.sort(expectedLines);
Collections.sort(actualLines);
assertEquals(expectedLines.size(), actualLines.size());
for (int i = 0; i < expectedLines.size(); i++) {
assertEquals(expectedLines.get(i), actualLines.get(i));
}
}

private void verifyFieldOrder(final String[] expectedFields, final String query)
throws IOException {

Expand Down
5 changes: 3 additions & 2 deletions integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE;
import static org.opensearch.sql.util.TestUtils.assertRowsEqual;

import java.io.IOException;
import java.util.Locale;
Expand All @@ -27,7 +28,7 @@ public void sanitizeTest() throws IOException {
Locale.ROOT,
"source=%s | fields firstname, lastname",
TEST_INDEX_BANK_CSV_SANITIZE));
assertEquals(
assertRowsEqual(
StringUtils.format(
"firstname,lastname%n"
+ "'+Amber JOHnny,Duke Willmington+%n"
Expand All @@ -47,7 +48,7 @@ public void escapeSanitizeTest() throws IOException {
"source=%s | fields firstname, lastname",
TEST_INDEX_BANK_CSV_SANITIZE),
false);
assertEquals(
assertRowsEqual(
StringUtils.format(
"firstname,lastname%n"
+ "+Amber JOHnny,Duke Willmington+%n"
Expand Down
61 changes: 60 additions & 1 deletion integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;

Expand All @@ -35,7 +38,16 @@ public void testConsecutiveDedup() throws IOException {
executeQuery(
String.format(
"source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK));
verifyDataRows(result, rows(true), rows(false), rows(true), rows(false));
List<Object[]> actualRows = extractActualRows(result);
List<Object[]> expectedRows = getExpectedDedupRows(actualRows);
assertTrue("Deduplication was not consecutive", expectedRows != null);
assertEquals(
"Row count after deduplication does not match", expectedRows.size(), actualRows.size());

// Verify the expected and actual rows match
for (int i = 0; i < expectedRows.size(); i++) {
assertArrayEquals(expectedRows.get(i), actualRows.get(i));
}
}

@Test
Expand All @@ -62,4 +74,51 @@ public void testKeepEmptyDedup() throws IOException {
rows("Virginia", null),
rows("Dillard", 48086));
}

private List<Object[]> extractActualRows(JSONObject result) {
JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> actualRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
actualRows.add(new Object[] {row.get(0)});
}
return actualRows;
}

// Create the expected deduplicated rows
private List<Object[]> getExpectedDedupRows(List<Object[]> actualRows) {
if (verifyConsecutiveDeduplication(actualRows)) {
return createExpectedRows(actualRows);
}
return null;
}

// Verify consecutive deduplication
private boolean verifyConsecutiveDeduplication(List<Object[]> actualRows) {
Object previousValue = null;

for (Object[] currentRow : actualRows) {
Object currentValue = currentRow[0];
if (previousValue != null && currentValue.equals(previousValue)) {
return false; // If consecutive values are the same, deduplication fails
}
previousValue = currentValue;
}
return true;
}

// Create the expected rows after deduplication
private List<Object[]> createExpectedRows(List<Object[]> actualRows) {
List<Object[]> expectedRows = new ArrayList<>();
Object previousValue = null;

for (Object[] currentRow : actualRows) {
Object currentValue = currentRow[0];
if (previousValue == null || !currentValue.equals(previousValue)) {
expectedRows.add(currentRow);
}
previousValue = currentValue;
}
return expectedRows;
}
}
114 changes: 85 additions & 29 deletions integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.verifyOrder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
import org.junit.Test;

public class ParseCommandIT extends PPLIntegTestCase {
Expand All @@ -26,15 +28,23 @@ public void testParseCommand() throws IOException {
executeQuery(
String.format(
"source=%s | parse email '.+@(?<host>.+)' | fields email, host", TEST_INDEX_BANK));
verifyOrder(
result,
rows("amberduke@pyrami.com", "pyrami.com"),
rows("hattiebond@netagy.com", "netagy.com"),
rows("nanettebates@quility.com", "quility.com"),
rows("daleadams@boink.com", "boink.com"),
rows("elinorratliff@scentric.com", "scentric.com"),
rows("virginiaayala@filodyne.com", "filodyne.com"),
rows("dillardmcpherson@quailcom.com", "quailcom.com"));

// Create the expected rows
List<Object[]> expectedRows =
new ArrayList<>(
List.of(
new Object[] {"amberduke@pyrami.com", "pyrami.com"},
new Object[] {"hattiebond@netagy.com", "netagy.com"},
new Object[] {"nanettebates@quility.com", "quility.com"},
new Object[] {"daleadams@boink.com", "boink.com"},
new Object[] {"elinorratliff@scentric.com", "scentric.com"},
new Object[] {"virginiaayala@filodyne.com", "filodyne.com"},
new Object[] {"dillardmcpherson@quailcom.com", "quailcom.com"}));

List<Object[]> actualRows = convertJsonToRows(result, 2);
sortRowsByFirstColumn(expectedRows);
sortRowsByFirstColumn(actualRows);
compareRows(expectedRows, actualRows);
}

@Test
Expand All @@ -43,15 +53,23 @@ public void testParseCommandReplaceOriginalField() throws IOException {
executeQuery(
String.format(
"source=%s | parse email '.+@(?<email>.+)' | fields email", TEST_INDEX_BANK));
verifyOrder(
result,
rows("pyrami.com"),
rows("netagy.com"),
rows("quility.com"),
rows("boink.com"),
rows("scentric.com"),
rows("filodyne.com"),
rows("quailcom.com"));

// Create the expected rows
List<Object[]> expectedRows =
new ArrayList<>(
List.of(
new Object[] {"pyrami.com"},
new Object[] {"netagy.com"},
new Object[] {"quility.com"},
new Object[] {"boink.com"},
new Object[] {"scentric.com"},
new Object[] {"filodyne.com"},
new Object[] {"quailcom.com"}));

List<Object[]> actualRows = convertJsonToRows(result, 1);
sortRowsByFirstColumn(expectedRows);
sortRowsByFirstColumn(actualRows);
compareRows(expectedRows, actualRows);
}

@Test
Expand All @@ -62,14 +80,52 @@ public void testParseCommandWithOtherRunTimeFields() throws IOException {
"source=%s | parse email '.+@(?<host>.+)' | "
+ "eval eval_result=1 | fields host, eval_result",
TEST_INDEX_BANK));
verifyOrder(
result,
rows("pyrami.com", 1),
rows("netagy.com", 1),
rows("quility.com", 1),
rows("boink.com", 1),
rows("scentric.com", 1),
rows("filodyne.com", 1),
rows("quailcom.com", 1));

// Create the expected rows as List<Object[]>
List<Object[]> expectedRows =
new ArrayList<>(
List.of(
new Object[] {"pyrami.com", 1},
new Object[] {"netagy.com", 1},
new Object[] {"quility.com", 1},
new Object[] {"boink.com", 1},
new Object[] {"scentric.com", 1},
new Object[] {"filodyne.com", 1},
new Object[] {"quailcom.com", 1}));

List<Object[]> actualRows = convertJsonToRows(result, 2);
sortRowsByFirstColumn(expectedRows);
sortRowsByFirstColumn(actualRows);
compareRows(expectedRows, actualRows);
}

// Convert JSON response to List<Object[]>
private List<Object[]> convertJsonToRows(JSONObject result, int columnCount) {
JSONArray dataRows = result.getJSONArray("datarows");
List<Object[]> rows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
Object[] rowData = new Object[columnCount];
for (int j = 0; j < columnCount; j++) {
rowData[j] = row.get(j);
}
rows.add(rowData);
}
return rows;
}

// Sort rows by the first column
private void sortRowsByFirstColumn(List<Object[]> rows) {
rows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));
}

private void compareRows(List<Object[]> expectedRows, List<Object[]> actualRows) {
if (expectedRows.size() != actualRows.size()) {
Assert.fail(
"Row count is different. expectedRows:" + expectedRows + ", actualRows: " + actualRows);
}
for (int i = 0; i < expectedRows.size(); i++) {
assertArrayEquals(expectedRows.get(i), actualRows.get(i));
}
}
}
74 changes: 70 additions & 4 deletions integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@
import static org.opensearch.sql.util.MatcherUtils.verifyOrder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Test;

Expand All @@ -38,17 +44,77 @@ public void testSortWithNullValue() throws IOException {
String.format(
"source=%s | sort balance | fields firstname, balance",
TEST_INDEX_BANK_WITH_NULL_VALUES));

JSONArray dataRows = result.getJSONArray("datarows");

// Filter null balance rows
List<Object[]> nullRows = filterRows(dataRows, 1, true);

// Verify the set values for null balances as rows with null balance can return in any order
List<Object[]> expectedNullRows =
Arrays.asList(
new Object[] {"Hattie", null},
new Object[] {"Elinor", null},
new Object[] {"Virginia", null});
assertSetEquals(expectedNullRows, nullRows);

// Filter non-null balance rows and create filtered result
List<Object[]> nonNullRows = filterRows(dataRows, 1, false);
JSONObject filteredResult = createFilteredResult(result, nonNullRows);

verifyOrder(
result,
rows("Hattie", null),
rows("Elinor", null),
rows("Virginia", null),
filteredResult,
rows("Dale", 4180),
rows("Nanette", 32838),
rows("Amber JOHnny", 39225),
rows("Dillard", 48086));
}

private void assertSetEquals(List<Object[]> expected, List<Object[]> actual) {
Set<List<Object>> expectedSet = new HashSet<>();
for (Object[] arr : expected) {
expectedSet.add(Arrays.asList(arr));
}

Set<List<Object>> actualSet = new HashSet<>();
for (Object[] arr : actual) {
actualSet.add(Arrays.asList(arr));
}

assertEquals(expectedSet, actualSet);
}

// Filter rows by null or non-null values based on the specified column index
private List<Object[]> filterRows(JSONArray dataRows, int columnIndex, boolean isNull) {
List<Object[]> filteredRows = new ArrayList<>();
for (int i = 0; i < dataRows.length(); i++) {
JSONArray row = dataRows.getJSONArray(i);
if ((isNull && row.isNull(columnIndex)) || (!isNull && !row.isNull(columnIndex))) {
Object[] rowData = new Object[row.length()];
for (int j = 0; j < row.length(); j++) {
rowData[j] = row.isNull(j) ? null : row.get(j);
}
filteredRows.add(rowData);
}
}
return filteredRows;
}

// Create a new JSONObject with filtered rows and updated metadata
private JSONObject createFilteredResult(JSONObject originalResult, List<Object[]> filteredRows) {
JSONArray jsonArray = new JSONArray();
for (Object[] row : filteredRows) {
jsonArray.put(new JSONArray(row));
}

JSONObject filteredResult = new JSONObject();
filteredResult.put("schema", originalResult.getJSONArray("schema"));
filteredResult.put("total", jsonArray.length());
filteredResult.put("datarows", jsonArray);
filteredResult.put("size", jsonArray.length());
return filteredResult;
}

@Test
public void testSortStringField() throws IOException {
JSONObject result =
Expand Down
Loading

0 comments on commit 6972487

Please sign in to comment.