Skip to content

Commit

Permalink
Support common format geo point
Browse files Browse the repository at this point in the history 
Signed-off-by: panguixin <panguixin@bytedance.com>
  • Loading branch information
@bugmakerrrrrr
bugmakerrrrrr committed Jul 3, 2024
1 parent 4326396 commit 53471f7
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 61 deletions.
39 changes: 20 additions & 19 deletions opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,19 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Iterators;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.OpenSearchParseException;
import org.opensearch.common.geo.GeoPoint;
import org.opensearch.common.geo.GeoUtils;
import org.opensearch.common.xcontent.json.JsonXContentParser;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;

/** The Implementation of Content to represent {@link JsonNode}. */
@RequiredArgsConstructor
Expand Down Expand Up @@ -122,25 +130,18 @@ public Object objectValue() {
@Override
public Pair<Double, Double> geoValue() {
final JsonNode value = value();
if (value.has("lat") && value.has("lon")) {
Double lat = 0d;
Double lon = 0d;
try {
lat = extractDoubleValue(value.get("lat"));
} catch (Exception exception) {
throw new IllegalStateException(
"latitude must be number value, but got value: " + value.get("lat"));
}
try {
lon = extractDoubleValue(value.get("lon"));
} catch (Exception exception) {
throw new IllegalStateException(
"longitude must be number value, but got value: " + value.get("lon"));
}
return Pair.of(lat, lon);
} else {
throw new IllegalStateException(
"geo point must in format of {\"lat\": number, \"lon\": number}");
try (XContentParser parser =
new JsonXContentParser(
NamedXContentRegistry.EMPTY,
DeprecationHandler.IGNORE_DEPRECATIONS,
value.traverse())) {
assert parser.currentToken() == null;
parser.nextToken();
GeoPoint point = new GeoPoint();
GeoUtils.parseGeoPoint(parser, point, true);
return Pair.of(point.getLat(), point.getLon());
} catch (IOException ex) {
throw new OpenSearchParseException("error parsing geo point", ex);
}
}

Expand Down
51 changes: 45 additions & 6 deletions ...ch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,6 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) {
.put(
OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip),
(c, dt) -> new OpenSearchExprIpValue(c.stringValue()))
.put(
OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint),
(c, dt) ->
new OpenSearchExprGeoPointValue(c.geoValue().getLeft(), c.geoValue().getRight()))
.put(
OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary),
(c, dt) -> new OpenSearchExprBinaryValue(c.stringValue()))
Expand Down Expand Up @@ -193,8 +189,11 @@ private ExprValue parse(
return ExprNullValue.of();
}

ExprType type = fieldType.get();
if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested))
final ExprType type = fieldType.get();

if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))) {
return parseGeoPoint(content, supportArrays);
} else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested))
|| content.isArray()) {
return parseArray(content, field, type, supportArrays);
} else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object))
Expand Down Expand Up @@ -363,6 +362,46 @@ private ExprValue parseArray(
return new ExprCollectionValue(result);
}

/**
* Parse geo point content.
*
* @param content Content to parse.
* @param supportArrays Parsing the whole array or not
* @return Geo point value parsed from content.
*/
private ExprValue parseGeoPoint(Content content, boolean supportArrays) {
// there is only one point in doc.
if (content.isArray() == false) {
final var pair = content.geoValue();
return new OpenSearchExprGeoPointValue(pair.getLeft(), pair.getRight());
}

var elements = content.array();
var first = elements.next();
// an array in the [longitude, latitude] format.
if (first.isNumber()) {
double lon = first.doubleValue();
double lat = elements.next().doubleValue();
return new OpenSearchExprGeoPointValue(lat, lon);
}

// there are multi points in doc
var pair = first.geoValue();
var firstPoint = new OpenSearchExprGeoPointValue(pair.getLeft(), pair.getRight());
if (supportArrays) {
List<ExprValue> result = new ArrayList<>();
result.add(firstPoint);
elements.forEachRemaining(
e -> {
var p = e.geoValue();
result.add(new OpenSearchExprGeoPointValue(p.getLeft(), p.getRight()));
});
return new ExprCollectionValue(result);
} else {
return firstPoint;
}
}

/**
* Parse inner array value. Can be object type and recurse continues.
*
Expand Down
73 changes: 37 additions & 36 deletions ...rc/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import lombok.EqualsAndHashCode;
import lombok.ToString;
import org.junit.jupiter.api.Test;
import org.opensearch.geometry.utils.Geohash;
import org.opensearch.sql.data.model.ExprCollectionValue;
import org.opensearch.sql.data.model.ExprDateValue;
import org.opensearch.sql.data.model.ExprTimeValue;
Expand Down Expand Up @@ -671,53 +672,53 @@ public void constructIP() {
tupleValue("{\"ipV\":\"192.168.0.1\"}").get("ipV"));
}

private static final double TOLERANCE = 1E-5;

@Test
public void constructGeoPoint() {
final double lat = 42.60355556;
final double lon = -97.25263889;
final var expectedGeoPointValue = new OpenSearchExprGeoPointValue(lat, lon);
// An object with a latitude and longitude.
assertEquals(
new OpenSearchExprGeoPointValue(42.60355556, -97.25263889),
tupleValue("{\"geoV\":{\"lat\":42.60355556,\"lon\":-97.25263889}}").get("geoV"));
assertEquals(
new OpenSearchExprGeoPointValue(42.60355556, -97.25263889),
tupleValue("{\"geoV\":{\"lat\":\"42.60355556\",\"lon\":\"-97.25263889\"}}").get("geoV"));
assertEquals(
new OpenSearchExprGeoPointValue(42.60355556, -97.25263889),
constructFromObject("geoV", "42.60355556,-97.25263889"));
}
expectedGeoPointValue,
tupleValue(String.format("{\"geoV\":{\"lat\":%.8f,\"lon\":%.8f}}", lat, lon)).get("geoV"));

@Test
public void constructGeoPointFromUnsupportedFormatShouldThrowException() {
IllegalStateException exception =
assertThrows(
IllegalStateException.class,
() -> tupleValue("{\"geoV\":[42.60355556,-97.25263889]}").get("geoV"));
// A string in the “latitude,longitude” format.
assertEquals(
"geo point must in format of {\"lat\": number, \"lon\": number}", exception.getMessage());
expectedGeoPointValue,
tupleValue(String.format("{\"geoV\":\"%.8f,%.8f\"}", lat, lon)).get("geoV"));

exception =
assertThrows(
IllegalStateException.class,
() -> tupleValue("{\"geoV\":{\"lon\":-97.25263889}}").get("geoV"));
// A geohash.
var point =
(OpenSearchExprGeoPointValue.GeoPoint)
tupleValue(String.format("{\"geoV\":\"%s\"}", Geohash.stringEncode(lon, lat)))
.get("geoV")
.value();
assertEquals(lat, point.getLat(), TOLERANCE);
assertEquals(lon, point.getLon(), TOLERANCE);

// An array in the [longitude, latitude] format.
assertEquals(
"geo point must in format of {\"lat\": number, \"lon\": number}", exception.getMessage());
expectedGeoPointValue,
tupleValue(String.format("{\"geoV\":[%.8f, %.8f]}", lon, lat)).get("geoV"));

exception =
assertThrows(
IllegalStateException.class,
() -> tupleValue("{\"geoV\":{\"lat\":-97.25263889}}").get("geoV"));
// A Well-Known Text POINT in the “POINT(longitude latitude)” format.
assertEquals(
"geo point must in format of {\"lat\": number, \"lon\": number}", exception.getMessage());
expectedGeoPointValue,
tupleValue(String.format("{\"geoV\":\"POINT (%.8f %.8f)\"}", lon, lat)).get("geoV"));

exception =
assertThrows(
IllegalStateException.class,
() -> tupleValue("{\"geoV\":{\"lat\":true,\"lon\":-97.25263889}}").get("geoV"));
assertEquals("latitude must be number value, but got value: true", exception.getMessage());
// GeoJSON format, where the coordinates are in the [longitude, latitude] format
assertEquals(
expectedGeoPointValue,
tupleValue(
String.format(
"{\"geoV\":{\"type\":\"Point\",\"coordinates\":[%.8f,%.8f]}}", lon, lat))
.get("geoV"));

exception =
assertThrows(
IllegalStateException.class,
() -> tupleValue("{\"geoV\":{\"lat\":42.60355556,\"lon\":false}}").get("geoV"));
assertEquals("longitude must be number value, but got value: false", exception.getMessage());
assertEquals(
new OpenSearchExprGeoPointValue(42.60355556, -97.25263889),
constructFromObject("geoV", "42.60355556,-97.25263889"));
}

@Test
Expand Down

0 comments on commit 53471f7

Please sign in to comment.