From 9825e040ea1e6e7e4cae2ee908b50555f586b565 Mon Sep 17 00:00:00 2001 From: Guillaume Smet Date: Thu, 24 Nov 2016 14:41:02 +0100 Subject: [PATCH] Support Elasticsearch permissive syntax for sort and _source --- .../main/java/io/searchbox/core/Search.java | 87 ++++++++++--- .../java/io/searchbox/core/SearchTest.java | 114 ++++++++++++++++-- 2 files changed, 173 insertions(+), 28 deletions(-) diff --git a/jest-common/src/main/java/io/searchbox/core/Search.java b/jest-common/src/main/java/io/searchbox/core/Search.java index 283a5e19d..93d4013ae 100644 --- a/jest-common/src/main/java/io/searchbox/core/Search.java +++ b/jest-common/src/main/java/io/searchbox/core/Search.java @@ -10,7 +10,10 @@ import com.google.gson.Gson; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSyntaxException; import io.searchbox.action.AbstractAction; import io.searchbox.action.AbstractMultiTypeActionBuilder; @@ -90,26 +93,16 @@ public String getData(Gson gson) { queryObject = new JsonObject(); } - JsonArray sortArray; - if (queryObject.has( "sort" )) { - sortArray = queryObject.get("sort").getAsJsonArray(); - } else { - sortArray = new JsonArray(); - queryObject.add("sort", sortArray); - } + if (!sortList.isEmpty()) { + JsonArray sortArray = normalizeSortClause(queryObject); - for (Sort sort : sortList) { - sortArray.add(sort.toJsonObject()); + for (Sort sort : sortList) { + sortArray.add(sort.toJsonObject()); + } } if (!includePatternList.isEmpty() || !excludePatternList.isEmpty()) { - JsonObject sourceObject; - if (queryObject.has("_source")) { - sourceObject = queryObject.get("_source").getAsJsonObject(); - } else { - sourceObject = new JsonObject(); - queryObject.add("_source", sourceObject); - } + JsonObject sourceObject = normalizeSourceClause(queryObject); addPatternListToSource(sourceObject, "include", includePatternList); addPatternListToSource(sourceObject, "exclude", excludePatternList); @@ -120,6 +113,68 @@ public String getData(Gson gson) { return data; } + private static JsonArray normalizeSortClause(JsonObject queryObject) { + JsonArray sortArray; + if (queryObject.has("sort")) { + JsonElement sortElement = queryObject.get("sort"); + if (sortElement.isJsonArray()) { + sortArray = sortElement.getAsJsonArray(); + } else if (sortElement.isJsonObject()) { + sortArray = new JsonArray(); + sortArray.add(sortElement.getAsJsonObject()); + } else if (sortElement.isJsonPrimitive() && sortElement.getAsJsonPrimitive().isString()) { + String sortField = sortElement.getAsString(); + sortArray = new JsonArray(); + queryObject.add("sort", sortArray); + String order; + if ("_score".equals(sortField)) { + order = "desc"; + } else { + order = "asc"; + } + JsonObject sortOrder = new JsonObject(); + sortOrder.add("order", new JsonPrimitive(order)); + JsonObject sortDefinition = new JsonObject(); + sortDefinition.add(sortField, sortOrder); + + sortArray.add(sortDefinition); + } else { + throw new JsonSyntaxException("_source must be an array, an object or a string"); + } + } else { + sortArray = new JsonArray(); + } + queryObject.add("sort", sortArray); + + return sortArray; + } + + private static JsonObject normalizeSourceClause(JsonObject queryObject) { + JsonObject sourceObject; + if (queryObject.has("_source")) { + JsonElement sourceElement = queryObject.get("_source"); + + if (sourceElement.isJsonObject()) { + sourceObject = sourceElement.getAsJsonObject(); + } else if (sourceElement.isJsonArray()) { + // in this case, the values of the array are includes + sourceObject = new JsonObject(); + queryObject.add("_source", sourceObject); + sourceObject.add("include", sourceElement.getAsJsonArray()); + } else if (sourceElement.isJsonPrimitive() && sourceElement.getAsJsonPrimitive().isBoolean()) { + // if _source is a boolean, we override the configuration with include/exclude + sourceObject = new JsonObject(); + } else { + throw new JsonSyntaxException("_source must be an object, an array or a boolean"); + } + } else { + sourceObject = new JsonObject(); + } + queryObject.add("_source", sourceObject); + + return sourceObject; + } + private static void addPatternListToSource(JsonObject sourceObject, String rule, List patternList) { if (!patternList.isEmpty()) { JsonArray ruleArray; diff --git a/jest-common/src/test/java/io/searchbox/core/SearchTest.java b/jest-common/src/test/java/io/searchbox/core/SearchTest.java index de00d8450..83fd137c8 100644 --- a/jest-common/src/test/java/io/searchbox/core/SearchTest.java +++ b/jest-common/src/test/java/io/searchbox/core/SearchTest.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; +import java.util.List; import org.junit.Test; @@ -146,6 +147,52 @@ public void sourceFilteringParamTest() { assertEquals(excludePatternItem2, excludePattern.get(1).getAsString()); } + @Test + public void supportElasticsearchPermissiveSourceFilteringSyntax() { + String query = "{\"query\" : { \"term\" : { \"name\" : \"KangSungJeon\" } }, \"_source\": false}"; + String includePatternItem1 = "SeolaIncludeFieldName"; + String excludePatternItem1 = "SeolaExcludeField.*"; + + Action search = new Search.Builder(query) + .addSourceIncludePattern(includePatternItem1) + .addSourceExcludePattern(excludePatternItem1) + .build(); + + JsonParser parser = new JsonParser(); + JsonElement parsed = parser.parse(search.getData(new Gson()).toString()); + JsonObject obj = parsed.getAsJsonObject(); + JsonObject source = obj.getAsJsonObject("_source"); + + JsonArray includePattern = source.getAsJsonArray("include"); + assertEquals(1, includePattern.size()); + assertEquals(includePatternItem1, includePattern.get(0).getAsString()); + + JsonArray excludePattern = source.getAsJsonArray("exclude"); + assertEquals(1, excludePattern.size()); + assertEquals(excludePatternItem1, excludePattern.get(0).getAsString()); + + query = "{\"query\" : { \"term\" : { \"name\" : \"KangSungJeon\" } }, \"_source\": [\"includeFieldName1\", \"includeFieldName2\"]}"; + + search = new Search.Builder(query) + .addSourceIncludePattern(includePatternItem1) + .addSourceExcludePattern(excludePatternItem1) + .build(); + + parsed = parser.parse(search.getData(new Gson()).toString()); + obj = parsed.getAsJsonObject(); + source = obj.getAsJsonObject("_source"); + + includePattern = source.getAsJsonArray("include"); + assertEquals(3, includePattern.size()); + assertEquals("includeFieldName1", includePattern.get(0).getAsString()); + assertEquals("includeFieldName2", includePattern.get(1).getAsString()); + assertEquals(includePatternItem1, includePattern.get(2).getAsString()); + + excludePattern = source.getAsJsonArray("exclude"); + assertEquals(1, excludePattern.size()); + assertEquals(excludePatternItem1, excludePattern.get(0).getAsString()); + } + @Test public void sortTest() { String query = "{\"query\" : { \"term\" : { \"name\" : \"Milano\" } }}"; @@ -186,21 +233,53 @@ public void sortTest() { @Test public void addSortShouldNotOverrideExistingSortDefinitions() { - String query = "{\"query\" : { \"term\" : { \"name\" : \"Milano\" } }, \"sort\": [{\"existing\": { \"order\": \"desc\" }}]}"; - Action search = new Search.Builder(query) - .addSort(Arrays.asList(sortByPopulationAsc, sortByPopulationDesc)).build(); + JsonArray sortClause = buildSortClause( + "{\"query\" : { \"term\" : { \"name\" : \"Milano\" } }, \"sort\": [{\"existing\": { \"order\": \"desc\" }}]}", + Arrays.asList(sortByPopulationAsc, sortByPopulationDesc) + ); - JsonParser parser = new JsonParser(); - JsonElement parsed = parser.parse(search.getData(new Gson())); - JsonObject obj = parsed.getAsJsonObject(); - JsonArray sort = obj.getAsJsonArray("sort"); + assertNotNull(sortClause); + assertEquals(3, sortClause.size()); - assertNotNull(sort); - assertEquals(3, sort.size()); + assertEquals("{\"existing\":{\"order\":\"desc\"}}", sortClause.get(0).toString()); + assertEquals("{\"population\":{\"order\":\"asc\"}}", sortClause.get(1).toString()); + assertEquals("{\"population\":{\"order\":\"desc\"}}", sortClause.get(2).toString()); + } + + @Test + public void supportElasticsearchPermissiveSortSyntax() { + JsonArray sortClause = buildSortClause( + "{\"query\" : { \"term\" : { \"name\" : \"Milano\" } }, \"sort\": \"existing\"}", + Arrays.asList(sortByPopulationAsc) + ); - assertEquals("{\"existing\":{\"order\":\"desc\"}}", sort.get(0).toString()); - assertEquals("{\"population\":{\"order\":\"asc\"}}", sort.get(1).toString()); - assertEquals("{\"population\":{\"order\":\"desc\"}}", sort.get(2).toString()); + assertNotNull(sortClause); + assertEquals(2, sortClause.size()); + + assertEquals("{\"existing\":{\"order\":\"asc\"}}", sortClause.get(0).toString()); + assertEquals("{\"population\":{\"order\":\"asc\"}}", sortClause.get(1).toString()); + + sortClause = buildSortClause( + "{\"query\" : { \"term\" : { \"name\" : \"Milano\" } }, \"sort\": \"_score\"}", + Arrays.asList(sortByPopulationAsc) + ); + + assertNotNull(sortClause); + assertEquals(2, sortClause.size()); + + assertEquals("{\"_score\":{\"order\":\"desc\"}}", sortClause.get(0).toString()); + assertEquals("{\"population\":{\"order\":\"asc\"}}", sortClause.get(1).toString()); + + sortClause = buildSortClause( + "{\"query\" : { \"term\" : { \"name\" : \"Milano\" } }, \"sort\": { \"existing\": {\"order\":\"desc\"} }}", + Arrays.asList(sortByPopulationAsc) + ); + + assertNotNull(sortClause); + assertEquals(2, sortClause.size()); + + assertEquals("{\"existing\":{\"order\":\"desc\"}}", sortClause.get(0).toString()); + assertEquals("{\"population\":{\"order\":\"asc\"}}", sortClause.get(1).toString()); } @Test @@ -238,4 +317,15 @@ public void equalsReturnsFalseForDifferentSortList() { assertNotEquals(search1, search1Duplicate); } + + private JsonArray buildSortClause(String query, List sorts) { + Action search = new Search.Builder(query).addSort(sorts).build(); + + JsonParser parser = new JsonParser(); + Gson gson = new Gson(); + JsonElement parsed = parser.parse(search.getData(gson)); + JsonObject obj = parsed.getAsJsonObject(); + + return obj.getAsJsonArray("sort"); + } }