Skip to content

Commit

Permalink
[DerivedField] object type support in mappings and new settings for d…
Browse files Browse the repository at this point in the history 
…erived field (#13717)


---------

Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
  • Loading branch information
@rishabhmaurya
rishabhmaurya authored Jun 3, 2024
1 parent 6c1896b commit bb013da
Show file tree
Hide file tree
Showing 3 changed files with 172 additions and 4 deletions.
81 changes: 77 additions & 4 deletions server/src/main/java/org/opensearch/index/mapper/DerivedField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.opensearch.index.mapper;

import org.opensearch.Version;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
Expand All @@ -18,17 +19,21 @@
import org.opensearch.script.Script;

import java.io.IOException;
import java.util.Map;
import java.util.Objects;

/**
* DerivedField representation: expects a name, type and script.
*/
@PublicApi(since = "2.14.0")
public class DerivedField implements Writeable, ToXContentFragment {

private final String name;
private final String type;
private final Script script;
private String sourceIndexedField;
private Map<String, Object> properties;
private Boolean ignoreMalformed;
private String format;

public DerivedField(String name, String type, Script script) {
this.name = name;
Expand All @@ -40,20 +45,51 @@ public DerivedField(StreamInput in) throws IOException {
name = in.readString();
type = in.readString();
script = new Script(in);
if (in.getVersion().onOrAfter(Version.V_2_15_0)) {
if (in.readBoolean()) {
properties = in.readMap();
}
sourceIndexedField = in.readOptionalString();
format = in.readOptionalString();
ignoreMalformed = in.readOptionalBoolean();
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
out.writeString(type);
script.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_2_15_0)) {
if (properties == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeMap(properties);
}
out.writeOptionalString(sourceIndexedField);
out.writeOptionalString(format);
out.writeOptionalBoolean(ignoreMalformed);
}
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(name);
builder.field("type", type);
builder.field("script", script);
if (properties != null) {
builder.field("properties", properties);
}
if (sourceIndexedField != null) {
builder.field("source_indexed_field", sourceIndexedField);
}
if (format != null) {
builder.field("format", format);
}
if (ignoreMalformed != null) {
builder.field("ignore_malformed", ignoreMalformed);
}
builder.endObject();
return builder;
}
Expand All @@ -70,9 +106,41 @@ public Script getScript() {
return script;
}

public Map<String, Object> getProperties() {
return properties;
}

public String getSourceIndexedField() {
return sourceIndexedField;
}

public String getFormat() {
return format;
}

public boolean getIgnoreMalformed() {
return Boolean.TRUE.equals(ignoreMalformed);
}

public void setProperties(Map<String, Object> properties) {
this.properties = properties;
}

public void setSourceIndexedField(String sourceIndexedField) {
this.sourceIndexedField = sourceIndexedField;
}

public void setFormat(String format) {
this.format = format;
}

public void setIgnoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
}

@Override
public int hashCode() {
return Objects.hash(name, type, script);
return Objects.hash(name, type, script, sourceIndexedField, properties, ignoreMalformed, format);
}

@Override
Expand All @@ -84,7 +152,12 @@ public boolean equals(Object obj) {
return false;
}
DerivedField other = (DerivedField) obj;
return Objects.equals(name, other.name) && Objects.equals(type, other.type) && Objects.equals(script, other.script);
return Objects.equals(name, other.name)
&& Objects.equals(type, other.type)
&& Objects.equals(script, other.script)
&& Objects.equals(sourceIndexedField, other.sourceIndexedField)
&& Objects.equals(properties, other.properties)
&& Objects.equals(ignoreMalformed, other.ignoreMalformed)
&& Objects.equals(format, other.format);
}

}
31 changes: 31 additions & 0 deletions server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1004,6 +1004,37 @@ public SearchSourceBuilder derivedField(String name, String type, Script script)
return this;
}

/**
* Adds a derived field with the given name with provided type, script and other parameters
* @param name name of the derived field
* @param type type of the derived field
* @param script script associated with derived field
* @param properties map of field name and type of field for nested fields within object derived field
* @param sourceIndexedField source text field which is indexed to filter documents for better performance
* @param format date format
* @param ignoreMalformed ignores malformed fields instead of failing search request
*/
public SearchSourceBuilder derivedField(
String name,
String type,
Script script,
Map<String, Object> properties,
String sourceIndexedField,
String format,
Boolean ignoreMalformed
) {
if (derivedFields == null) {
derivedFields = new ArrayList<>();
}
DerivedField derivedField = new DerivedField(name, type, script);
derivedField.setProperties(properties);
derivedField.setSourceIndexedField(sourceIndexedField);
derivedField.setFormat(format);
derivedField.setIgnoreMalformed(ignoreMalformed);
derivedFields.add(derivedField);
return this;
}

/**
* Sets the boost a specific index or alias will receive when the query is executed
* against it.
Expand Down
64 changes: 64 additions & 0 deletions server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,70 @@ public void testDerivedFieldsParsingAndSerialization() throws IOException {

}

public void testDerivedFieldsParsingAndSerializationObjectType() throws IOException {
{
String restContent = "{\n"
+ " \"derived\": {\n"
+ " \"duration\": {\n"
+ " \"type\": \"long\",\n"
+ " \"script\": \"emit(doc['test'])\"\n"
+ " },\n"
+ " \"ip_from_message\": {\n"
+ " \"type\": \"keyword\",\n"
+ " \"script\": \"emit(doc['message'])\"\n"
+ " },\n"
+ " \"object\": {\n"
+ " \"type\": \"object\",\n"
+ " \"script\": \"emit(doc['test'])\",\n"
+ " \"format\": \"dd-MM-yyyy\",\n"
+ " \"source_indexed_field\": \"test\",\n"
+ " \"ignore_malformed\": true,\n"
+ " \"properties\": {\n"
+ " \"sub_field\": \"text\"\n"
+ " }\n"
+ " }\n"
+ " },\n"
+ " \"query\" : {\n"
+ " \"match\": { \"content\": { \"query\": \"foo bar\" }}\n"
+ " }\n"
+ "}";

String expectedContent =
"{\"query\":{\"match\":{\"content\":{\"query\":\"foo bar\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},\"derived\":{\"duration\":{\"type\":\"long\",\"script\":\"emit(doc['test'])\"},\"ip_from_message\":{\"type\":\"keyword\",\"script\":\"emit(doc['message'])\"},\"object\":{\"format\":\"dd-MM-yyyy\",\"source_indexed_field\":\"test\",\"ignore_malformed\":true,\"type\":\"object\",\"script\":\"emit(doc['test'])\",\"properties\":{\"sub_field\":\"text\"}},\"derived_field\":{\"type\":\"object\",\"script\":{\"source\":\"emit(doc['message']\",\"lang\":\"painless\"},\"properties\":{\"sub_field_2\":\"keyword\"},\"source_indexed_field\":\"message\",\"format\":\"dd-MM-yyyy\",\"ignore_malformed\":true}}}";

try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
searchSourceBuilder.derivedField(
"derived_field",
"object",
new Script("emit(doc['message']"),
Map.of("sub_field_2", "keyword"),
"message",
"dd-MM-yyyy",
true
);
searchSourceBuilder = rewrite(searchSourceBuilder);
assertEquals(3, searchSourceBuilder.getDerivedFieldsObject().size());
assertEquals(1, searchSourceBuilder.getDerivedFields().size());
assertEquals(1, searchSourceBuilder.getDerivedFields().get(0).getProperties().size());
assertEquals("message", searchSourceBuilder.getDerivedFields().get(0).getSourceIndexedField());
assertEquals("dd-MM-yyyy", searchSourceBuilder.getDerivedFields().get(0).getFormat());
assertTrue(searchSourceBuilder.getDerivedFields().get(0).getIgnoreMalformed());

try (BytesStreamOutput output = new BytesStreamOutput()) {
searchSourceBuilder.writeTo(output);
try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
String actualContent = deserializedBuilder.toString();
assertEquals(expectedContent, actualContent);
assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode());
assertNotSame(searchSourceBuilder, deserializedBuilder);
}
}
}
}
}

public void testAggsParsing() throws IOException {
{
String restContent = "{\n"
Expand Down

0 comments on commit bb013da

Please sign in to comment.