Skip to content

Commit

Permalink
Updating Ip fields to use doc_values to search (#11508) (#12078)
Browse files Browse the repository at this point in the history
* Updating Ip fields to use doc_values to search



* Fix IP tests



* Fix skip to allow yaml test to pass on main



* Update tests to use existing test file



* Changing skip version to match bwc



* Using exact match instead of range



* Spotless



* Fix IP field tests



* Fix spotless + precommit failure



* Get point out of query and into value



* Fix term tests



* Add skip test logic to only doc_values test



---------


(cherry picked from commit cc9ee9d)

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
1 parent 7e3c080 commit de636c1
Show file tree
Hide file tree
Showing 4 changed files with 323 additions and 48 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
- Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877)), ([#12000](https://github.com/opensearch-project/OpenSearch/pull/12000))
- Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968))
- Updates IpField to be searchable when only `doc_values` are enabled ([#11508](https://github.com/opensearch-project/OpenSearch/pull/11508))

### Deprecated

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,18 +47,22 @@ setup:
type: unsigned_long
index: true
doc_values: true
ip_field:
type: ip
index: true
doc_values: true

- do:
bulk:
index: test-iodvq
refresh: true
body:
- '{"index": {"_index": "test-iodvq", "_id": "1" }}'
- '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }'
- '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
- '{ "index": { "_index": "test-iodvq", "_id": "2" }}'
- '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }'
- '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
- '{ "index": { "_index": "test-iodvq", "_id": "3" } }'
- '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }'
- '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'

- do:
search:
Expand Down Expand Up @@ -162,7 +166,6 @@ setup:

- match: { hits.total: 1 }


- do:
search:
rest_total_hits_as_int: true
Expand All @@ -174,6 +177,16 @@ setup:

- match: { hits.total: 1 }

- do:
search:
rest_total_hits_as_int: true
index: test-iodvq
body:
query:
term:
ip_field: "192.168.0.1"

- match: {hits.total: 1}

- do:
search:
Expand All @@ -186,7 +199,6 @@ setup:

- match: { hits.total: 2 }


- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -264,6 +276,17 @@ setup:

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-iodvq
body:
query:
terms:
ip_field: ["192.168.0.1", "192.168.0.2"]

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -384,6 +407,19 @@ setup:

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-iodvq
body:
query:
range:
ip_field:
gte: "192.168.0.1"
lte: "192.168.0.2"

- match: { hits.total: 2 }

---
"search on fields with only index enabled":
- do:
Expand Down Expand Up @@ -428,18 +464,22 @@ setup:
type: unsigned_long
index: true
doc_values: false
ip_field:
type: ip
index: true
doc_values: false

- do:
bulk:
index: test-index
refresh: true
body:
- '{"index": {"_index": "test-index", "_id": "1" }}'
- '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }'
- '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
- '{ "index": { "_index": "test-index", "_id": "2" }}'
- '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }'
- '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
- '{ "index": { "_index": "test-index", "_id": "3" } }'
- '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }'
- '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'

- do:
search:
Expand All @@ -465,7 +505,6 @@ setup:

- match: { hits.total: 2 }


- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -555,6 +594,16 @@ setup:

- match: { hits.total: 1 }

- do:
search:
rest_total_hits_as_int: true
index: test-index
body:
query:
term:
ip_field: "192.168.0.1"

- match: {hits.total: 1}

- do:
search:
Expand All @@ -567,7 +616,6 @@ setup:

- match: { hits.total: 2 }


- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -645,6 +693,17 @@ setup:

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-index
body:
query:
terms:
ip_field: ["192.168.0.1", "192.168.0.2"]

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -765,8 +824,24 @@ setup:

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-index
body:
query:
range:
ip_field:
gte: "192.168.0.1"
lte: "192.168.0.2"

- match: { hits.total: 2 }
---
"search on fields with only doc_values enabled":
- skip:
features: [ "headers" ]
version: " - 2.99.99"
reason: "searching with only doc_values was added in 3.0.0"
- do:
indices.create:
index: test-doc-values
Expand Down Expand Up @@ -809,18 +884,22 @@ setup:
type: unsigned_long
index: false
doc_values: true
ip_field:
type: ip
index: false
doc_values: true

- do:
bulk:
index: test-doc-values
refresh: true
body:
- '{"index": {"_index": "test-doc-values", "_id": "1" }}'
- '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }'
- '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
- '{ "index": { "_index": "test-doc-values", "_id": "2" }}'
- '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }'
- '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
- '{ "index": { "_index": "test-doc-values", "_id": "3" } }'
- '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }'
- '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'

- do:
search:
Expand All @@ -846,7 +925,6 @@ setup:

- match: { hits.total: 2 }


- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -924,7 +1002,6 @@ setup:

- match: { hits.total: 1 }


- do:
search:
rest_total_hits_as_int: true
Expand All @@ -936,6 +1013,16 @@ setup:

- match: { hits.total: 1 }

- do:
search:
rest_total_hits_as_int: true
index: test-doc-values
body:
query:
term:
ip_field: "192.168.0.3"

- match: { hits.total: 1 }

- do:
search:
Expand All @@ -948,7 +1035,6 @@ setup:

- match: { hits.total: 2 }


- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -1145,3 +1231,16 @@ setup:
}

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-doc-values
body:
query:
range:
ip_field:
gte: "192.168.0.1"
lte: "192.168.0.2"

- match: { hits.total: 2 }
67 changes: 55 additions & 12 deletions server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -222,25 +224,48 @@ protected Object parseSourceValue(Object value) {

@Override
public Query termQuery(Object value, @Nullable QueryShardContext context) {
failIfNotIndexed();
failIfNotIndexedAndNoDocValues();
Query query;
if (value instanceof InetAddress) {
return InetAddressPoint.newExactQuery(name(), (InetAddress) value);
query = InetAddressPoint.newExactQuery(name(), (InetAddress) value);
} else {
if (value instanceof BytesRef) {
value = ((BytesRef) value).utf8ToString();
}
String term = value.toString();
if (term.contains("/")) {
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
} else {
InetAddress address = InetAddresses.forString(term);
query = InetAddressPoint.newExactQuery(name(), address);
}
}
if (isSearchable() && hasDocValues()) {
String term = value.toString();
if (term.contains("/")) {
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
}
return new IndexOrDocValuesQuery(
query,
SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()))
);
}
if (hasDocValues()) {
String term = value.toString();
if (term.contains("/")) {
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
}
InetAddress address = InetAddresses.forString(term);
return InetAddressPoint.newExactQuery(name(), address);
return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()));
}
return query;
}

@Override
public Query termsQuery(List<?> values, QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
InetAddress[] addresses = new InetAddress[values.size()];
int i = 0;
for (Object value : values) {
Expand All @@ -265,14 +290,32 @@ public Query termsQuery(List<?> values, QueryShardContext context) {

@Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
failIfNotIndexed();
return rangeQuery(
lowerTerm,
upperTerm,
includeLower,
includeUpper,
(lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper)
);
failIfNotIndexedAndNoDocValues();
return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> {
Query query = InetAddressPoint.newRangeQuery(name(), lower, upper);
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(
query,
SortedSetDocValuesField.newSlowRangeQuery(
((PointRangeQuery) query).getField(),
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
true,
true
)
);
}
if (hasDocValues()) {
return SortedSetDocValuesField.newSlowRangeQuery(
((PointRangeQuery) query).getField(),
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
true,
true
);
}
return query;
});
}

/**
Expand Down
Loading

0 comments on commit de636c1

Please sign in to comment.