From 2e676472738dbe3e7ce49ef64aa0fd4df0fc1f4b Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Tue, 30 Jul 2024 11:30:06 -0700 Subject: [PATCH] more rest tests Signed-off-by: bowenlan-amzn --- .../test/search/175_terms_lookup.yml | 67 ------- .../test/search/370_bitmap_filtering.yml | 179 ++++++++++++++++++ .../index/query/TermsQueryBuilder.java | 7 + 3 files changed, 186 insertions(+), 67 deletions(-) delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/175_terms_lookup.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/175_terms_lookup.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/175_terms_lookup.yml deleted file mode 100644 index 9201835acac6f..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/175_terms_lookup.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -"Terms lookup on a bitmap field": - - do: - indices.create: - index: students - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - student_id: - type: integer - - do: - bulk: - refresh: true - body: - - { "index": { "_index": "students", "_id": "1" } } - - { "name": "Jane Doe", "student_id": 111 } - - { "index": { "_index": "students", "_id": "2" } } - - { "name": "Mary Major", "student_id": 222 } - - { "index": { "_index": "students", "_id": "3" } } - - { "name": "John Doe", "student_id": 333 } - - do: - indices.create: - index: classes - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - enrolled: - type: binary - store: true - - do: - index: - index: classes - id: 101 - body: { "enrolled" : "OjAAAAEAAAAAAAEAEAAAAG8A3gA=" } - - - do: - cluster.health: - wait_for_status: green - - - do: - search: - rest_total_hits_as_int: true - index: students - body: { - "query": { - "terms": { - "student_id": { - "index": "classes", - "id": "101", - "path": "enrolled", - "store": true - }, - "value_type": "bitmap" - } - } - } - - match: { hits.total: 2 } - - match: { hits.hits.0._source.name: Jane Doe } - - match: { hits.hits.0._source.student_id: 111 } - - match: { hits.hits.1._source.name: Mary Major } - - match: { hits.hits.1._source.student_id: 222 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml new file mode 100644 index 0000000000000..3332fe0a1fe26 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml @@ -0,0 +1,179 @@ +--- +setup: + - do: + indices.create: + index: students + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + student_id: + type: integer + - do: + bulk: + refresh: true + body: + - { "index": { "_index": "students", "_id": "1" } } + - { "name": "Jane Doe", "student_id": 111 } + - { "index": { "_index": "students", "_id": "2" } } + - { "name": "Mary Major", "student_id": 222 } + - { "index": { "_index": "students", "_id": "3" } } + - { "name": "John Doe", "student_id": 333 } + - do: + indices.create: + index: classes + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + enrolled: + type: binary + store: true + - do: + bulk: + refresh: true + body: + - { "index": { "_index": "classes", "_id": "101" } } + - { "enrolled": "OjAAAAEAAAAAAAEAEAAAAG8A3gA=" } # 111,222 + - { "index": { "_index": "classes", "_id": "102" } } + - { "enrolled": "OjAAAAEAAAAAAAAAEAAAAG8A" } # 111 + - { "index": { "_index": "classes", "_id": "103" } } + - { "enrolled": "OjAAAAEAAAAAAAAAEAAAAE0B" } # 333 + - { "index": { "_index": "classes", "_id": "104" } } + - { "enrolled": "OjAAAAEAAAAAAAEAEAAAAN4ATQE=" } # 222,333 + - do: + cluster.health: + wait_for_status: green + +--- +"Terms lookup on a binary field with bitmap": + - do: + search: + rest_total_hits_as_int: true + index: students + body: { + "query": { + "terms": { + "student_id": { + "index": "classes", + "id": "101", + "path": "enrolled", + "store": true + }, + "value_type": "bitmap" + } + } + } + - match: { hits.total: 2 } + - match: { hits.hits.0._source.name: Jane Doe } + - match: { hits.hits.0._source.student_id: 111 } + - match: { hits.hits.1._source.name: Mary Major } + - match: { hits.hits.1._source.student_id: 222 } +--- +"Terms query accepting bitmap as value": + - do: + search: + rest_total_hits_as_int: true + index: students + body: { + "query": { + "terms": { + "student_id": "OjAAAAEAAAAAAAEAEAAAAG8A3gA=", + "value_type": "bitmap" + } + } + } + - match: { hits.total: 2 } + - match: { hits.hits.0._source.name: Jane Doe } + - match: { hits.hits.0._source.student_id: 111 } + - match: { hits.hits.1._source.name: Mary Major } + - match: { hits.hits.1._source.student_id: 222 } +--- +"Boolean must bitmap filtering": + - do: + search: + rest_total_hits_as_int: true + index: students + body: { + "query": { + "bool": { + "must": [ + { + "terms": { + "student_id": { + "index": "classes", + "id": "101", + "path": "enrolled", + "store": true + }, + "value_type": "bitmap" + } + } + ], + "must_not": [ + { + "terms": { + "student_id": { + "index": "classes", + "id": "104", + "path": "enrolled", + "store": true + }, + "value_type": "bitmap" + } + } + ] + } + } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._source.name: Jane Doe } + - match: { hits.hits.0._source.student_id: 111 } + +--- +"Boolean should bitmap filtering": + - do: + search: + rest_total_hits_as_int: true + index: students + body: { + "query": { + "bool": { + "should": [ + { + "terms": { + "student_id": { + "index": "classes", + "id": "101", + "path": "enrolled", + "store": true + }, + "value_type": "bitmap" + } + }, + { + "terms": { + "student_id": { + "index": "classes", + "id": "104", + "path": "enrolled", + "store": true + }, + "value_type": "bitmap" + } + } + ] + } + } + } + - match: { hits.total: 3 } + - match: { hits.hits.0._source.name: Jane Doe } + - match: { hits.hits.0._source.student_id: 111 } + - match: { hits.hits.0._source.name: Mary Major } + - match: { hits.hits.0._source.student_id: 222 } + - match: { hits.hits.0._source.name: John Doe } + - match: { hits.hits.0._source.student_id: 333 } diff --git a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java index 1c47d7a7c2c37..efa8ba3b7f88c 100644 --- a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java @@ -64,6 +64,7 @@ import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -462,6 +463,12 @@ public static TermsQueryBuilder fromXContent(XContentParser parser) throws IOExc ); } + // if value_type is not empty, and values is a BytesRef, + // we parse the bytes to the corresponding structure here + if (valueType != null && values != null && values.size() == 1 && values.get(0) instanceof BytesRef) { + values.set(0, new BytesArray(Base64.getDecoder().decode(((BytesRef) values.get(0)).utf8ToString()))); + } + return new TermsQueryBuilder(fieldName, values, termsLookup).boost(boost).queryName(queryName).valueType(valueType); }