Skip to content

Commit

Permalink
Use lenient match queries to allow mixed data fields with wrong value…
Browse files Browse the repository at this point in the history
…s provided in the query
  • Loading branch information
carlosdelest committed Dec 5, 2024
1 parent fa14ff5 commit 9060cba
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -547,4 +547,15 @@ hire_date_nanos:date_nanos
1986-06-26T00:00:00.000Z
;

testMatchWithWrongFieldValue
required_capability: match_function
required_capability: match_additional_types

from employees,employees_incompatible
| where match(still_hired::boolean, "Wrong boolean")
| eval emp_no_bool = emp_no::boolean
| keep emp_no_bool
;

emp_no_bool:boolean
;
Original file line number Diff line number Diff line change
Expand Up @@ -568,3 +568,16 @@ hire_date_nanos:date_nanos
1986-06-26T00:00:00.000Z
;

testMatchWithWrongFieldValue
required_capability: match_function
required_capability: match_additional_types

from employees,employees_incompatible
| where still_hired::boolean : "Wrong boolean"
| eval emp_no_bool = emp_no::boolean
| keep emp_no_bool
;

emp_no_bool:boolean
;

Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,8 @@ protected Query asQuery(Match match, TranslatorHandler handler) {
// If we have multiple field types, we allow the query to be done, but getting the underlying field name
fieldName = multiTypeEsField.getName();
}
return new MatchQuery(match.source(), fieldName, match.queryAsObject());
// Make query lenient so mixed field types can be queried when a field type is incompatible with the value provided
return new MatchQuery(match.source(), fieldName, match.queryAsObject(), Map.of("lenient", "true"));
}

throw new IllegalArgumentException("Match must have a field attribute as the first argument");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,7 @@ public void testMatchFunction() {
var field = as(project.child(), FieldExtractExec.class);
var query = as(field.child(), EsQueryExec.class);
assertThat(query.limit().fold(), is(1000));
var expected = QueryBuilders.matchQuery("last_name", "Smith");
var expected = QueryBuilders.matchQuery("last_name", "Smith").lenient(true);
assertThat(query.query().toString(), is(expected.toString()));
}

Expand Down Expand Up @@ -678,7 +678,7 @@ public void testMatchFunctionConjunctionWhereOperands() {

Source filterSource = new Source(2, 38, "emp_no > 10000");
var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
var queryString = QueryBuilders.matchQuery("last_name", "Smith");
var queryString = QueryBuilders.matchQuery("last_name", "Smith").lenient(true);
var expected = QueryBuilders.boolQuery().must(queryString).must(range);
assertThat(query.query().toString(), is(expected.toString()));
}
Expand Down Expand Up @@ -713,7 +713,7 @@ public void testMatchFunctionWithFunctionsPushedToLucene() {

Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")");
var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource);
var queryString = QueryBuilders.matchQuery("text", "beta");
var queryString = QueryBuilders.matchQuery("text", "beta").lenient(true);
var expected = QueryBuilders.boolQuery().must(queryString).must(terms);
assertThat(query.query().toString(), is(expected.toString()));
}
Expand Down Expand Up @@ -747,7 +747,7 @@ public void testMatchFunctionMultipleWhereClauses() {

Source filterSource = new Source(3, 8, "emp_no > 10000");
var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
var queryString = QueryBuilders.matchQuery("last_name", "Smith");
var queryString = QueryBuilders.matchQuery("last_name", "Smith").lenient(true);
var expected = QueryBuilders.boolQuery().must(queryString).must(range);
assertThat(query.query().toString(), is(expected.toString()));
}
Expand Down Expand Up @@ -777,8 +777,8 @@ public void testMatchFunctionMultipleMatchClauses() {
var query = as(field.child(), EsQueryExec.class);
assertThat(query.limit().fold(), is(1000));

var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith");
var queryStringRight = QueryBuilders.matchQuery("first_name", "John");
var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith").lenient(true);
var queryStringRight = QueryBuilders.matchQuery("first_name", "John").lenient(true);
var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight);
assertThat(query.query().toString(), is(expected.toString()));
}
Expand Down Expand Up @@ -1476,7 +1476,7 @@ private void checkMatchFunctionPushDown(
var fieldExtract = as(project.child(), FieldExtractExec.class);
var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query();

var expectedLuceneQuery = new MatchQueryBuilder(fieldName, expectedValueProvider.apply(queryValue));
var expectedLuceneQuery = new MatchQueryBuilder(fieldName, expectedValueProvider.apply(queryValue)).lenient(true);
assertThat("Unexpected match query for data type " + fieldDataType, actualLuceneQuery, equalTo(expectedLuceneQuery));
} catch (ParsingException e) {
fail("Error parsing ESQL query: " + esqlQuery + "\n" + e.getMessage());
Expand Down Expand Up @@ -1551,10 +1551,10 @@ public void testMultipleMatchFilterPushdown() {
var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query();

Source filterSource = new Source(4, 8, "emp_no > 10000");
var expectedLuceneQuery = new BoolQueryBuilder().must(new MatchQueryBuilder("first_name", "Anna"))
.must(new MatchQueryBuilder("first_name", "Anneke"))
var expectedLuceneQuery = new BoolQueryBuilder().must(new MatchQueryBuilder("first_name", "Anna").lenient(true))
.must(new MatchQueryBuilder("first_name", "Anneke").lenient(true))
.must(wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10000), "emp_no", filterSource))
.must(new MatchQueryBuilder("last_name", "Xinglin"));
.must(new MatchQueryBuilder("last_name", "Xinglin").lenient(true));
assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString()));
}

Expand Down

0 comments on commit 9060cba

Please sign in to comment.