Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

$in with empty array should find nothing #626

Merged
merged 5 commits into from
Nov 10, 2023
Merged

$in with empty array should find nothing #626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dae9b49
$in with empty array should find nothing
Yuqi-Du Nov 9, 2023
5f51337
unit test
Yuqi-Du Nov 9, 2023
a8cf25a
fix test
Yuqi-Du Nov 9, 2023
713850c
clean
Yuqi-Du Nov 9, 2023
ada0804
Merge remote-tracking branch 'origin/main' into fix_in_and_or
Yuqi-Du Nov 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/ExpressionBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,18 @@
import io.stargate.sgv2.jsonapi.exception.ErrorCode;
import io.stargate.sgv2.jsonapi.exception.JsonApiException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

public class ExpressionBuilder {

public static List<Expression<BuiltCondition>> buildExpressions(
LogicalExpression logicalExpression, DBFilterBase.IDFilter additionalIdFilter) {
// an empty filter should find everything
if (logicalExpression.isEmpty() && additionalIdFilter == null) {
return Collections.singletonList(null);
}
// after validate in FilterClauseDeserializer,
// partition key column key will not be nested under OR operator
// so we can collect all id_conditions, then do a combination to generate separate queries
Expand Down Expand Up @@ -43,8 +48,7 @@ private static List<Expression<BuiltCondition>> buildExpressionWithId(
if (idFilters.isEmpty()
&& additionalIdFilter == null) { // no idFilters in filter clause and no additionalIdFilter
if (expressionWithoutId == null) {
expressionsWithId.add(null); // should find everything
return expressionsWithId;
return null; // should find nothing
} else {
return List.of(expressionWithoutId);
}
Expand Down Expand Up @@ -91,12 +95,17 @@ private static Expression<BuiltCondition> buildExpressionRecursive(
}
conditionExpressions.add(subExpressionCondition);
}

boolean hasInFilterThisLevel = false;
boolean inFilterThisLevelWithEmptyArray = true;
// second for loop, is to iterate all subComparisonExpression
for (ComparisonExpression comparisonExpression : logicalExpression.comparisonExpressions) {
for (DBFilterBase dbFilter : comparisonExpression.getDbFilters()) {
if (dbFilter instanceof DBFilterBase.InFilter inFilter) {
hasInFilterThisLevel = true;
List<BuiltCondition> inFilterConditions = inFilter.getAll();
if (!inFilterConditions.isEmpty()) {
inFilterThisLevelWithEmptyArray = false;
List<Variable<BuiltCondition>> inConditionsVariables =
inFilterConditions.stream().map(Variable::of).toList();
conditionExpressions.add(ExpressionUtils.orOf(inConditionsVariables));
Expand All @@ -115,6 +124,14 @@ private static Expression<BuiltCondition> buildExpressionRecursive(
if (conditionExpressions.isEmpty()) {
return null;
}
// when having an empty array $in, if $in occurs within an $and logic, entire $and should match
// nothing
if (hasInFilterThisLevel
tatu-at-datastax marked this conversation as resolved.
Show resolved Hide resolved
&& inFilterThisLevelWithEmptyArray
&& logicalExpression.getLogicalRelation().equals(LogicalExpression.LogicalOperator.AND)) {
return null;
}

return ExpressionUtils.buildExpression(
conditionExpressions, logicalExpression.getLogicalRelation().getOperator());
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ public ReadDocument getNewDocument() {
private List<QueryOuterClass.Query> buildSelectQueries(DBFilterBase.IDFilter additionalIdFilter) {
List<Expression<BuiltCondition>> expressions =
ExpressionBuilder.buildExpressions(logicalExpression, additionalIdFilter);
if (expressions == null) { // in filter, but with empty values, find nothing
if (expressions == null) { // find nothing
return List.of();
}
List<QueryOuterClass.Query> queries = new ArrayList<>(expressions.size());
Expand Down Expand Up @@ -501,7 +501,7 @@ private List<QueryOuterClass.Query> buildSortedSelectQueries(
DBFilterBase.IDFilter additionalIdFilter) {
List<Expression<BuiltCondition>> expressions =
ExpressionBuilder.buildExpressions(logicalExpression, additionalIdFilter);
if (expressions == null) {
if (expressions == null) { // find nothing
return List.of();
}
String[] columns = sortedDataColumns;
Expand Down
98 changes: 97 additions & 1 deletion src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ public void inConditionWithOtherCondition() {
}

@Test
public void inConditionEmptyArray() {
public void idInConditionEmptyArray() {
String json =
"""
{
Expand All @@ -393,6 +393,102 @@ public void inConditionEmptyArray() {
.body("errors", is(nullValue()));
}

@Test
public void nonIDInConditionEmptyArray() {
String json =
"""
{
"find": {
"filter" : {
"username" : {"$in" : []}
}
}
}
""";
given()
.header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken())
.contentType(ContentType.JSON)
.body(json)
.when()
.post(CollectionResource.BASE_PATH, namespaceName, collectionName)
.then()
.statusCode(200)
.body("data.documents", hasSize(0))
.body("status", is(nullValue()))
.body("errors", is(nullValue()));
}

@Test
public void nonIDInConditionEmptyArrayAnd() {
String json =
"""
{
"find": {
"filter" : {
"$and": [
{
"age": {
"$in": []
}
},
{
"username": "user1"
}
]
}
}
}
""";
given()
.header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken())
.contentType(ContentType.JSON)
.body(json)
.when()
.post(CollectionResource.BASE_PATH, namespaceName, collectionName)
.then()
.statusCode(200)
.body("data.documents", hasSize(0))
.body("status", is(nullValue()))
.body("errors", is(nullValue()));
}

@Test
public void nonIDInConditionEmptyArrayOr() {
String json =
"""
{
"find": {
"filter" : {
"$or": [
{
"age": {
"$in": []
}
},
{
"username": "user1"
}
]
}
}
}
""";
String expected1 =
"{\"_id\":\"doc1\", \"username\":\"user1\", \"active_user\":true, \"date\" : {\"$date\": 1672531200000}}";
given()
.header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken())
.contentType(ContentType.JSON)
.body(json)
.when()
.post(CollectionResource.BASE_PATH, namespaceName, collectionName)
.then()
.statusCode(200)
.body("data.documents", hasSize(1))
.body("status", is(nullValue()))
.body("errors", is(nullValue()))
.body("data.documents[0]", jsonEquals(expected1));
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be better to have a test case that returns no results here, because there's no way of telling whether the document expected1 matched the first condition in $or, the 2nd, or both.

}

@Test
public void inOperatorEmptyArrayWithAdditionalFilters() {
String json =
Expand Down
31 changes: 0 additions & 31 deletions src/test/java/io/stargate/sgv2/jsonapi/service/operation/model/impl/FindOperationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,6 @@ public void findAll() throws Exception {
Values.of(doc2))));

LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.unsorted(
COMMAND_CONTEXT,
Expand Down Expand Up @@ -1537,10 +1533,6 @@ public void findAllSort() throws Exception {
Values.NULL)));

LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.sorted(
COMMAND_CONTEXT,
Expand Down Expand Up @@ -1755,10 +1747,6 @@ public void findAllSortByDate() throws Exception {
Values.of(1672531600000L))));

LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.sorted(
COMMAND_CONTEXT,
Expand Down Expand Up @@ -1954,10 +1942,6 @@ public void findAllSortWithSkip() throws Exception {
Values.NULL)));

LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.sorted(
COMMAND_CONTEXT,
Expand Down Expand Up @@ -2151,10 +2135,6 @@ public void findAllSortDescending() throws Exception {
Values.NULL)));

LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.sorted(
COMMAND_CONTEXT,
Expand Down Expand Up @@ -2551,10 +2531,6 @@ public void vectorSearch() throws Exception {
Values.of(doc2))));

LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.vsearch(
VECTOR_COMMAND_CONTEXT,
Expand Down Expand Up @@ -2671,9 +2647,6 @@ public void vectorSearchWithSimilarityProjection() throws Exception {
}
"""));
LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.vsearch(
Expand Down Expand Up @@ -2793,10 +2766,6 @@ public void vectorSearchWithSimilarityProjectionDotProduct() throws Exception {
}
"""));
LogicalExpression implicitAnd = LogicalExpression.and();
implicitAnd.comparisonExpressions.add(new ComparisonExpression(null, null, null));
List<DBFilterBase> filters = List.of();
implicitAnd.comparisonExpressions.get(0).setDBFilters(filters);

FindOperation operation =
FindOperation.vsearch(
VECTOR_DOT_PRODUCT_COMMAND_CONTEXT,
Expand Down