From 97d5f8946717de5b641f43e7af0a538d2a21ad2f Mon Sep 17 00:00:00 2001 From: guqing Date: Fri, 2 Feb 2024 17:52:01 +0800 Subject: [PATCH] refactor: only select the required index fields when build query index view # Conflicts: # api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java # application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java --- .../extension/index/query/LogicalQuery.java | 2 + .../extension/index/query/QueryFactory.java | 23 ++++++ .../extension/index/query/SimpleQuery.java | 2 + .../index/query/QueryFactoryTest.java | 78 +++++++++++++++---- .../index/IndexedQueryEngineImpl.java | 23 ++++++ 5 files changed, 112 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java b/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java index 4d81a1ef8e..2311109b5f 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java @@ -2,7 +2,9 @@ import java.util.Collection; import java.util.Objects; +import lombok.Getter; +@Getter public abstract class LogicalQuery implements Query { protected final Collection childQueries; protected final int size; diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java b/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java index 8ac83ef2f5..741d96c508 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import lombok.experimental.UtilityClass; import org.springframework.util.Assert; @@ -207,4 +208,26 @@ public static Query endsWith(String fieldName, String value) { public static Query contains(String fieldName, String value) { return new StringContains(fieldName, value); } + + /** + * Get all the field names used in the given query. + * + * @param query the query + * @return the field names used in the given query + */ + public static List getFieldNamesUsedInQuery(Query query) { + List fieldNames = new ArrayList<>(); + + if (query instanceof SimpleQuery simpleQuery) { + if (simpleQuery.isFieldRef()) { + fieldNames.add(simpleQuery.getValue()); + } + fieldNames.add(simpleQuery.getFieldName()); + } else if (query instanceof LogicalQuery logicalQuery) { + for (Query childQuery : logicalQuery.getChildQueries()) { + fieldNames.addAll(getFieldNamesUsedInQuery(childQuery)); + } + } + return fieldNames; + } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java b/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java index da41561fe9..9310661616 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java @@ -1,8 +1,10 @@ package run.halo.app.extension.index.query; +import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.springframework.util.Assert; +@Getter public abstract class SimpleQuery implements Query { protected final String fieldName; protected final String value; diff --git a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java b/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java index acce9acd19..e7b8918d72 100644 --- a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java +++ b/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java @@ -2,11 +2,28 @@ import static org.assertj.core.api.Assertions.assertThat; import static run.halo.app.extension.index.query.QueryFactory.all; +import static run.halo.app.extension.index.query.QueryFactory.and; import static run.halo.app.extension.index.query.QueryFactory.between; +import static run.halo.app.extension.index.query.QueryFactory.contains; +import static run.halo.app.extension.index.query.QueryFactory.endsWith; import static run.halo.app.extension.index.query.QueryFactory.equal; import static run.halo.app.extension.index.query.QueryFactory.equalOtherField; +import static run.halo.app.extension.index.query.QueryFactory.getFieldNamesUsedInQuery; +import static run.halo.app.extension.index.query.QueryFactory.greaterThan; +import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqual; +import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqualOtherField; +import static run.halo.app.extension.index.query.QueryFactory.greaterThanOtherField; +import static run.halo.app.extension.index.query.QueryFactory.in; +import static run.halo.app.extension.index.query.QueryFactory.isNotNull; +import static run.halo.app.extension.index.query.QueryFactory.isNull; +import static run.halo.app.extension.index.query.QueryFactory.lessThan; +import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqual; +import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqualOtherField; +import static run.halo.app.extension.index.query.QueryFactory.lessThanOtherField; import static run.halo.app.extension.index.query.QueryFactory.notEqual; import static run.halo.app.extension.index.query.QueryFactory.notEqualOtherField; +import static run.halo.app.extension.index.query.QueryFactory.or; +import static run.halo.app.extension.index.query.QueryFactory.startsWith; import org.junit.jupiter.api.Test; @@ -30,7 +47,7 @@ void allTest() { @Test void isNullTest() { var indexView = IndexViewDataSet.createPostIndexViewWithNullCell(); - var resultSet = QueryFactory.isNull("publishTime").matches(indexView); + var resultSet = isNull("publishTime").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102", "103", "104", "108" ); @@ -39,7 +56,7 @@ void isNullTest() { @Test void isNotNullTest() { var indexView = IndexViewDataSet.createPostIndexViewWithNullCell(); - var resultSet = QueryFactory.isNotNull("publishTime").matches(indexView); + var resultSet = isNotNull("publishTime").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "105", "106", "107" ); @@ -85,7 +102,7 @@ void notEqualOtherFieldTest() { @Test void lessThanTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.lessThan("id", "103").matches(indexView); + var resultSet = lessThan("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102" ); @@ -94,7 +111,7 @@ void lessThanTest() { @Test void lessThanOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.lessThanOtherField("id", "managerId").matches(indexView); + var resultSet = lessThanOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101" ); @@ -103,7 +120,7 @@ void lessThanOtherFieldTest() { @Test void lessThanOrEqualTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.lessThanOrEqual("id", "103").matches(indexView); + var resultSet = lessThanOrEqual("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103" ); @@ -113,7 +130,7 @@ void lessThanOrEqualTest() { void lessThanOrEqualOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); var resultSet = - QueryFactory.lessThanOrEqualOtherField("id", "managerId").matches(indexView); + lessThanOrEqualOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103" ); @@ -122,7 +139,7 @@ void lessThanOrEqualOtherFieldTest() { @Test void greaterThanTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.greaterThan("id", "103").matches(indexView); + var resultSet = greaterThan("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -131,7 +148,7 @@ void greaterThanTest() { @Test void greaterThanOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.greaterThanOtherField("id", "managerId").matches(indexView); + var resultSet = greaterThanOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -140,7 +157,7 @@ void greaterThanOtherFieldTest() { @Test void greaterThanOrEqualTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.greaterThanOrEqual("id", "103").matches(indexView); + var resultSet = greaterThanOrEqual("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104", "105" ); @@ -150,7 +167,7 @@ void greaterThanOrEqualTest() { void greaterThanOrEqualOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); var resultSet = - QueryFactory.greaterThanOrEqualOtherField("id", "managerId").matches(indexView); + greaterThanOrEqualOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102", "103", "104", "105" ); @@ -159,7 +176,7 @@ void greaterThanOrEqualOtherFieldTest() { @Test void inTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.in("id", "103", "104").matches(indexView); + var resultSet = in("id", "103", "104").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104" ); @@ -168,7 +185,7 @@ void inTest() { @Test void inTest2() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.in("lastName", "Fay").matches(indexView); + var resultSet = in("lastName", "Fay").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "104", "105" ); @@ -221,7 +238,7 @@ void betweenExclusive() { @Test void startsWithTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.startsWith("firstName", "W").matches(indexView); + var resultSet = startsWith("firstName", "W").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" ); @@ -230,7 +247,7 @@ void startsWithTest() { @Test void endsWithTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.endsWith("firstName", "y").matches(indexView); + var resultSet = endsWith("firstName", "y").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103" ); @@ -239,11 +256,11 @@ void endsWithTest() { @Test void containsTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.contains("firstName", "i").matches(indexView); + var resultSet = contains("firstName", "i").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" ); - resultSet = QueryFactory.contains("firstName", "N").matches(indexView); + resultSet = contains("firstName", "N").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -258,4 +275,33 @@ void notTest() { "100", "101", "103", "104", "105" ); } + + @Test + void getUsedFieldNamesTest() { + // single query + var query = equal("firstName", "W"); + var fieldNames = getFieldNamesUsedInQuery(query); + assertThat(fieldNames).containsExactlyInAnyOrder("firstName"); + + // and composite query + query = and( + and(equal("firstName", "W"), equal("lastName", "Fay")), + or(equalOtherField("id", "userId"), lessThan("age", "123")) + ); + fieldNames = getFieldNamesUsedInQuery(query); + assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", "id", "userId", + "age"); + + // or composite query + var complexQuery = or( + equal("field1", "value1"), + and( + equal("field2", "value2"), + equal("field3", "value3") + ), + equal("field4", "value4") + ); + fieldNames = getFieldNamesUsedInQuery(complexQuery); + assertThat(fieldNames).containsExactlyInAnyOrder("field1", "field2", "field3", "field4"); + } } diff --git a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java index b3723fcb80..1e30d05d8a 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java @@ -4,12 +4,14 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Sort; @@ -20,6 +22,7 @@ import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; import run.halo.app.extension.PageRequest; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.extension.index.query.QueryIndexViewImpl; import run.halo.app.extension.router.selector.FieldSelector; import run.halo.app.extension.router.selector.LabelSelector; @@ -145,8 +148,12 @@ List doRetrieve(Indexer indexer, ListOptions options, Sort sort) { stopWatch.stop(); stopWatch.start("build index view"); + var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort); var indexViewMap = new HashMap>>(); for (Map.Entry entry : fieldPathEntryMap.entrySet()) { + if (!fieldNamesUsedInQuery.contains(entry.getKey())) { + continue; + } indexViewMap.put(entry.getKey(), entry.getValue().immutableEntries()); } // TODO optimize build indexView time @@ -183,6 +190,22 @@ List doRetrieve(Indexer indexer, ListOptions options, Sort sort) { return result; } + @NonNull + private Set getFieldNamesUsedInListOptions(ListOptions options, Sort sort) { + var fieldNamesUsedInQuery = new HashSet(); + fieldNamesUsedInQuery.add(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME); + for (Sort.Order order : sort) { + fieldNamesUsedInQuery.add(order.getProperty()); + } + var hasFieldSelector = hasFieldSelector(options.getFieldSelector()); + if (hasFieldSelector) { + var fieldQuery = options.getFieldSelector().query(); + var fieldNames = QueryFactory.getFieldNamesUsedInQuery(fieldQuery); + fieldNamesUsedInQuery.addAll(fieldNames); + } + return fieldNamesUsedInQuery; + } + boolean hasLabelSelector(LabelSelector labelSelector) { return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers()); }