Skip to content

Commit

Permalink
fix #2828 Add configurable DisMax query support for default field sea…
Browse files Browse the repository at this point in the history
…rches
  • Loading branch information
marevol committed Jul 11, 2024
1 parent a21cea6 commit 7ee6b77
Show file tree
Hide file tree
Showing 8 changed files with 425 additions and 17 deletions.
69 changes: 69 additions & 0 deletions src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -1110,6 +1110,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. true */
String QUERY_BOOST_FUZZY_CONTENT_TRANSPOSITIONS = "query.boost.fuzzy.content.transpositions";

/** The key of the configuration. e.g. bool */
String QUERY_DEFAULT_query_type = "query.default.query_type";

/** The key of the configuration. e.g. 0.1 */
String QUERY_DISMAX_tie_breaker = "query.dismax.tie_breaker";

/** The key of the configuration. e.g. */
String QUERY_BOOL_minimum_should_match = "query.bool.minimum_should_match";

/** The key of the configuration. e.g. 50 */
String QUERY_PREFIX_EXPANSIONS = "query.prefix.expansions";

Expand Down Expand Up @@ -5244,6 +5253,43 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
boolean isQueryBoostFuzzyContentTranspositions();

/**
* Get the value for the key 'query.default.query_type'. <br>
* The value is, e.g. bool <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryDefaultQueryType();

/**
* Get the value for the key 'query.dismax.tie_breaker'. <br>
* The value is, e.g. 0.1 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryDismaxTieBreaker();

/**
* Get the value for the key 'query.dismax.tie_breaker' as {@link java.math.BigDecimal}. <br>
* The value is, e.g. 0.1 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not decimal.
*/
java.math.BigDecimal getQueryDismaxTieBreakerAsDecimal();

/**
* Get the value for the key 'query.bool.minimum_should_match'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryBoolMinimumShouldMatch();

/**
* Get the value for the key 'query.bool.minimum_should_match' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getQueryBoolMinimumShouldMatchAsInteger();

/**
* Get the value for the key 'query.prefix.expansions'. <br>
* The value is, e.g. 50 <br>
Expand Down Expand Up @@ -9489,6 +9535,26 @@ public boolean isQueryBoostFuzzyContentTranspositions() {
return is(FessConfig.QUERY_BOOST_FUZZY_CONTENT_TRANSPOSITIONS);
}

public String getQueryDefaultQueryType() {
return get(FessConfig.QUERY_DEFAULT_query_type);
}

public String getQueryDismaxTieBreaker() {
return get(FessConfig.QUERY_DISMAX_tie_breaker);
}

public java.math.BigDecimal getQueryDismaxTieBreakerAsDecimal() {
return getAsDecimal(FessConfig.QUERY_DISMAX_tie_breaker);
}

public String getQueryBoolMinimumShouldMatch() {
return get(FessConfig.QUERY_BOOL_minimum_should_match);
}

public Integer getQueryBoolMinimumShouldMatchAsInteger() {
return getAsInteger(FessConfig.QUERY_BOOL_minimum_should_match);
}

public String getQueryPrefixExpansions() {
return get(FessConfig.QUERY_PREFIX_EXPANSIONS);
}
Expand Down Expand Up @@ -11158,6 +11224,9 @@ protected java.util.Map<String, String> prepareGeneratedDefaultMap() {
defaultMap.put(FessConfig.QUERY_BOOST_FUZZY_CONTENT_EXPANSIONS, "10");
defaultMap.put(FessConfig.QUERY_BOOST_FUZZY_CONTENT_prefix_length, "0");
defaultMap.put(FessConfig.QUERY_BOOST_FUZZY_CONTENT_TRANSPOSITIONS, "true");
defaultMap.put(FessConfig.QUERY_DEFAULT_query_type, "bool");
defaultMap.put(FessConfig.QUERY_DISMAX_tie_breaker, "0.1");
defaultMap.put(FessConfig.QUERY_BOOL_minimum_should_match, "");
defaultMap.put(FessConfig.QUERY_PREFIX_EXPANSIONS, "50");
defaultMap.put(FessConfig.QUERY_PREFIX_SLOP, "0");
defaultMap.put(FessConfig.QUERY_FUZZY_prefix_length, "0");
Expand Down
148 changes: 148 additions & 0 deletions src/main/java/org/codelibs/fess/query/DefaultQueryBuilder.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/*
* Copyright 2012-2024 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.query;

import java.io.IOException;
import java.util.Objects;

import org.apache.lucene.search.Query;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.DisMaxQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilderVisitor;
import org.opensearch.index.query.QueryRewriteContext;
import org.opensearch.index.query.QueryShardContext;

public class DefaultQueryBuilder implements QueryBuilder {

private final QueryBuilder queryBuilder;

private final QueryType queryType;

public DefaultQueryBuilder(final QueryBuilder queryBuilder) {
this.queryBuilder = queryBuilder;
if (queryBuilder instanceof BoolQueryBuilder) {
queryType = QueryType.BOOL;
} else if (queryBuilder instanceof DisMaxQueryBuilder) {
queryType = QueryType.DISMAX;
} else {
throw new IllegalArgumentException("Unknown query builder: " + queryBuilder);
}
}

public DefaultQueryBuilder add(final QueryBuilder innerQueryBuilder) {
switch (queryType) {
case BOOL:
((BoolQueryBuilder) queryBuilder).should(innerQueryBuilder);
break;
case DISMAX:
((DisMaxQueryBuilder) queryBuilder).add(innerQueryBuilder);
break;
default:
break;
}
return this;
}

enum QueryType {
BOOL, DISMAX;
}

@Override
public String getWriteableName() {
return queryBuilder.getWriteableName();
}

@Override
public Query toQuery(final QueryShardContext context) throws IOException {
return queryBuilder.toQuery(context);
}

@Override
public boolean isFragment() {
return queryBuilder.isFragment();
}

@Override
public QueryBuilder queryName(final String queryName) {
return queryBuilder.queryName(queryName);
}

@Override
public String queryName() {
return queryBuilder.queryName();
}

@Override
public float boost() {
return queryBuilder.boost();
}

@Override
public QueryBuilder boost(final float boost) {
return queryBuilder.boost(boost);
}

@Override
public String getName() {
return queryBuilder.getName();
}

@Override
public QueryBuilder rewrite(final QueryRewriteContext queryShardContext) throws IOException {
return queryBuilder.rewrite(queryShardContext);
}

@Override
public void visit(final QueryBuilderVisitor visitor) {
queryBuilder.visit(visitor);
}

@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
return queryBuilder.toXContent(builder, params);
}

@Override
public void writeTo(final StreamOutput out) throws IOException {
queryBuilder.writeTo(out);
}

@Override
public int hashCode() {
return queryBuilder.hashCode();
}

@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if ((obj == null) || (getClass() != obj.getClass())) {
return false;
}
final DefaultQueryBuilder other = (DefaultQueryBuilder) obj;
return Objects.equals(queryBuilder, other.queryBuilder);
}

@Override
public String toString() {
return queryBuilder.toString();
}

}
39 changes: 29 additions & 10 deletions src/main/java/org/codelibs/fess/query/QueryCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@
import java.lang.Character.UnicodeBlock;

import org.apache.lucene.search.Query;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.dbflute.optional.OptionalThing;
import org.lastaflute.web.util.LaRequestUtil;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.DisMaxQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.sort.SortBuilder;
Expand Down Expand Up @@ -72,30 +74,47 @@ protected OptionalThing<String[]> getQueryLanguages() {
(String[]) request.getAttribute(Constants.REQUEST_LANGUAGES)));
}

protected BoolQueryBuilder buildDefaultQueryBuilder(final FessConfig fessConfig, final QueryContext context,
protected DefaultQueryBuilder buildDefaultQueryBuilder(final FessConfig fessConfig, final QueryContext context,
final DefaultQueryBuilderFunction builder) {
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.should(builder.apply(fessConfig.getIndexFieldTitle(), fessConfig.getQueryBoostTitleAsDecimal().floatValue()));
boolQuery.should(builder.apply(fessConfig.getIndexFieldContent(), fessConfig.getQueryBoostContentAsDecimal().floatValue()));
final DefaultQueryBuilder defaultQuery = createDefaultQueryBuilder();
defaultQuery.add(builder.apply(fessConfig.getIndexFieldTitle(), fessConfig.getQueryBoostTitleAsDecimal().floatValue()));
defaultQuery.add(builder.apply(fessConfig.getIndexFieldContent(), fessConfig.getQueryBoostContentAsDecimal().floatValue()));
final float importantContentBoost = fessConfig.getQueryBoostImportantContentAsDecimal().floatValue();
if (importantContentBoost >= 0.0f) {
boolQuery.should(builder.apply(fessConfig.getIndexFieldImportantContent(), importantContentBoost));
defaultQuery.add(builder.apply(fessConfig.getIndexFieldImportantContent(), importantContentBoost));
}
final float importantContantLangBoost = fessConfig.getQueryBoostImportantContentLangAsDecimal().floatValue();
getQueryLanguages().ifPresent(langs -> stream(langs).of(stream -> stream.forEach(lang -> {
boolQuery.should(
defaultQuery.add(
builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig.getQueryBoostTitleLangAsDecimal().floatValue()));
boolQuery.should(builder.apply(fessConfig.getIndexFieldContent() + "_" + lang,
defaultQuery.add(builder.apply(fessConfig.getIndexFieldContent() + "_" + lang,
fessConfig.getQueryBoostContentLangAsDecimal().floatValue()));
if (importantContantLangBoost >= 0.0f) {
boolQuery.should(builder.apply(fessConfig.getIndexFieldImportantContent() + "_" + lang, importantContantLangBoost));
defaultQuery.add(builder.apply(fessConfig.getIndexFieldImportantContent() + "_" + lang, importantContantLangBoost));
}
})));
getQueryFieldConfig().additionalDefaultList.stream().forEach(f -> {
final QueryBuilder query = builder.apply(f.getFirst(), f.getSecond());
boolQuery.should(query);
defaultQuery.add(query);
});
return boolQuery;
return defaultQuery;
}

protected DefaultQueryBuilder createDefaultQueryBuilder() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();

if ("dismax".equals(fessConfig.getQueryDefaultQueryType())) {
final DisMaxQueryBuilder disMaxQuery = QueryBuilders.disMaxQuery();
disMaxQuery.tieBreaker(fessConfig.getQueryDismaxTieBreakerAsDecimal().floatValue());
return new DefaultQueryBuilder(disMaxQuery);
}

final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
final String minimumShouldMatch = fessConfig.getQueryBoolMinimumShouldMatch();
if (StringUtil.isNotBlank(minimumShouldMatch)) {
boolQuery.minimumShouldMatch(minimumShouldMatch);
}
return new DefaultQueryBuilder(boolQuery);
}

protected QueryBuilder buildMatchPhraseQuery(final String f, final String text) {
Expand Down
9 changes: 4 additions & 5 deletions src/main/java/org/codelibs/fess/query/TermQueryCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.core.message.UserMessages;
import org.opensearch.common.unit.Fuzziness;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.sort.SortOrder;
Expand Down Expand Up @@ -161,24 +160,24 @@ protected QueryBuilder convertDefaultTermQuery(final FessConfig fessConfig, fina
final float boost, final String field, final String text) {
context.addFieldLog(field, text);
context.addHighlightedQuery(text);
final BoolQueryBuilder boolQuery =
final DefaultQueryBuilder defaultQuery =
buildDefaultQueryBuilder(fessConfig, context, (f, b) -> buildMatchPhraseQuery(f, text).boost(b * boost));
final Integer fuzzyMinLength = fessConfig.getQueryBoostFuzzyMinLengthAsInteger();
if (fuzzyMinLength >= 0 && text.length() >= fuzzyMinLength) {
boolQuery.should(QueryBuilders.fuzzyQuery(fessConfig.getIndexFieldTitle(), text)
defaultQuery.add(QueryBuilders.fuzzyQuery(fessConfig.getIndexFieldTitle(), text)
.boost(fessConfig.getQueryBoostFuzzyTitleAsDecimal().floatValue())
.prefixLength(fessConfig.getQueryBoostFuzzyTitlePrefixLengthAsInteger())
.transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryBoostFuzzyTitleTranspositions()))
.fuzziness(Fuzziness.build(fessConfig.getQueryBoostFuzzyTitleFuzziness()))
.maxExpansions(fessConfig.getQueryBoostFuzzyTitleExpansionsAsInteger()));
boolQuery.should(QueryBuilders.fuzzyQuery(fessConfig.getIndexFieldContent(), text)
defaultQuery.add(QueryBuilders.fuzzyQuery(fessConfig.getIndexFieldContent(), text)
.prefixLength(fessConfig.getQueryBoostFuzzyContentPrefixLengthAsInteger())
.transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryBoostFuzzyContentTranspositions()))
.boost(fessConfig.getQueryBoostFuzzyContentAsDecimal().floatValue())
.fuzziness(Fuzziness.build(fessConfig.getQueryBoostFuzzyContentFuzziness()))
.maxExpansions(fessConfig.getQueryBoostFuzzyContentExpansionsAsInteger()));
}
return boolQuery;
return defaultQuery;
}

protected QueryBuilder convertSiteQuery(final FessConfig fessConfig, final QueryContext context, final TermQuery termQuery,
Expand Down
4 changes: 4 additions & 0 deletions src/main/resources/fess_config.properties
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,10 @@ query.boost.fuzzy.content.expansions=10
query.boost.fuzzy.content.prefix_length=0
query.boost.fuzzy.content.transpositions=true

query.default.query_type=bool
query.dismax.tie_breaker=0.1
query.bool.minimum_should_match=

query.prefix.expansions=50
query.prefix.slop=0
query.fuzzy.prefix_length=0
Expand Down
Loading

0 comments on commit 7ee6b77

Please sign in to comment.