Skip to content

Commit

Permalink
ingest: date_index_name processor template resolution
Browse files Browse the repository at this point in the history 
Review changes:
* Change to TemplateScript.Factory instead of ValueSource.wrap
* Add support for templated index_name_format
* Remove unecessary changes
  • Loading branch information
@jakelandis
jakelandis committed Jul 10, 2018
1 parent 86d6b1b commit f9a6ec1
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 72 deletions.
46 changes: 27 additions & 19 deletions ...s/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.ValueSource;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
Expand All @@ -44,23 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
public static final String TYPE = "date_index_name";

private final String field;
private final String indexNamePrefix;
private final String dateRounding;
private final String indexNameFormat;
private final TemplateScript.Factory indexNamePrefixTemplate;
private final TemplateScript.Factory dateRoundingTemplate;
private final TemplateScript.Factory indexNameFormatTemplate;
private final DateTimeZone timezone;
private final List<Function<String, DateTime>> dateFormats;
private final ScriptService scriptService;

DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
String indexNamePrefix, String dateRounding, String indexNameFormat, ScriptService scriptService) {
TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
TemplateScript.Factory indexNameFormatTemplate) {
super(tag);
this.field = field;
this.timezone = timezone;
this.dateFormats = dateFormats;
this.indexNamePrefix = indexNamePrefix;
this.dateRounding = dateRounding;
this.indexNameFormat = indexNameFormat;
this.scriptService = scriptService;
this.indexNamePrefixTemplate = indexNamePrefixTemplate;
this.dateRoundingTemplate = dateRoundingTemplate;
this.indexNameFormatTemplate = indexNameFormatTemplate;
}

@Override
Expand All @@ -87,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception {
if (dateTime == null) {
throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
}
String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate);
String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate);
String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate);

DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
StringBuilder builder = new StringBuilder()
Expand All @@ -98,7 +100,7 @@ public void execute(IngestDocument ingestDocument) throws Exception {
.append('}')
.append('>');
String dynamicIndexName = builder.toString();
ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), ValueSource.wrap(dynamicIndexName, scriptService));
ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName);
}

@Override
Expand All @@ -110,16 +112,16 @@ String getField() {
return field;
}

String getIndexNamePrefix() {
return indexNamePrefix;
TemplateScript.Factory getIndexNamePrefixTemplate() {
return indexNamePrefixTemplate;
}

String getDateRounding() {
return dateRounding;
TemplateScript.Factory getDateRoundingTemplate() {
return dateRoundingTemplate;
}

String getIndexNameFormat() {
return indexNameFormat;
TemplateScript.Factory getIndexNameFormatTemplate() {
return indexNameFormatTemplate;
}

DateTimeZone getTimezone() {
Expand Down Expand Up @@ -164,10 +166,16 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St

String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
TemplateScript.Factory indexNamePrefixTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService);
String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
TemplateScript.Factory dateRoundingTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService);
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix,
dateRounding, indexNameFormat, scriptService);
TemplateScript.Factory indexNameFormatTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate,
dateRoundingTemplate, indexNameFormatTemplate);
}
}

Expand Down
11 changes: 6 additions & 5 deletions ...ngest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.joda.time.DateTimeZone;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

Expand All @@ -40,9 +41,9 @@ public void testDefaults() throws Exception {
DateIndexNameProcessor processor = factory.create(null, null, config);
assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
assertThat(processor.getField(), Matchers.equalTo("_field"));
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
assertThat(processor.getDateRounding(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
}

Expand All @@ -64,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception {
config.put("index_name_format", "yyyyMMdd");

processor = factory.create(null, null, config);
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd"));

config = new HashMap<>();
config.put("field", "_field");
Expand All @@ -81,7 +82,7 @@ public void testSpecifyOptionalSettings() throws Exception {
config.put("date_rounding", "y");

processor = factory.create(null, null, config);
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix"));
}

public void testRequiredFields() throws Exception {
Expand Down
67 changes: 34 additions & 33 deletions ...est-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,35 +20,24 @@

import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.joda.time.format.DateTimeFormat;

import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;

import static org.elasticsearch.script.Script.DEFAULT_TEMPLATE_LANG;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class DateIndexNameProcessorTests extends ESTestCase {

public void testJodaPattern() throws Exception {
Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
DateIndexNameProcessor processor = new DateIndexNameProcessor(
"_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
"events-", "y", "yyyyMMdd", TestTemplateService.instance()
);

DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z"));
processor.execute(document);
Expand All @@ -57,8 +46,8 @@ public void testJodaPattern() throws Exception {

public void testTAI64N()throws Exception {
Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
dateProcessor.execute(document);
Expand All @@ -67,8 +56,8 @@ public void testTAI64N()throws Exception {

public void testUnixMs()throws Exception {
Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "1000500"));
dateProcessor.execute(document);
Expand All @@ -82,29 +71,41 @@ public void testUnixMs()throws Exception {

public void testUnix()throws Exception {
Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "1000.5"));
dateProcessor.execute(document);
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
}

public void testTemplateSupported() throws Exception {
ScriptService scriptService = mock(ScriptService.class);
TestTemplateService.MockTemplateScript.Factory factory = new TestTemplateService.MockTemplateScript.Factory("script_result");
when(scriptService.compile(any(Script.class), Matchers.<ScriptContext<TemplateScript.Factory>>any())).thenReturn(factory);
when(scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG)).thenReturn(true);
public void testTemplatedFields() throws Exception {
String indexNamePrefix = randomAlphaOfLength(10);
String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s");
String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy");
String date = Integer.toString(randomInt());
Function<String, DateTime> dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);

DateIndexNameProcessor dateProcessor = createProcessor("_field",
Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix,
dateRounding, indexNameFormat);

DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field",
Collections.singletonList(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null)),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", scriptService);
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
Collections.singletonMap("_field", "1000.5"));
Collections.singletonMap("_field", date));
dateProcessor.execute(document);

// here we only care that the script was compiled and that it returned what we expect.
Mockito.verify(scriptService).compile(any(Script.class), Matchers.<ScriptContext<TemplateScript.Factory>>any());
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("script_result"));
assertThat(document.getSourceAndMetadata().get("_index"),
equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat)
.print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>"));
}

private DateIndexNameProcessor createProcessor(String field, List<Function<String, DateTime>> dateFormats,
DateTimeZone timezone, String indexNamePrefix, String dateRounding,
String indexNameFormat) {
return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone,
new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix),
new TestTemplateService.MockTemplateScript.Factory(dateRounding),
new TestTemplateService.MockTemplateScript.Factory(indexNameFormat)
);
}
}
15 changes: 0 additions & 15 deletions server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -406,21 +406,6 @@ public void setFieldValue(String path, Object value) {
setFieldValue(path, value, false);
}

/**
* Sets the provided value to the provided path in the document.
* Any non existing path element will be created.
* If the last item in the path is a list, the value will replace the existing list as a whole.
* Use {@link #appendFieldValue(String, Object)} to append values to lists instead.
* @param path The path within the document in dot-notation
* @param valueSource The value source that will produce the value or values to append to the existing ones
* @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the
* item identified by the provided path.
*/
public void setFieldValue(String path, ValueSource valueSource) {
Map<String, Object> model = valueSource == null ? null : createTemplateModel();
setFieldValue(path, valueSource == null ? null : valueSource.copyAndResolve(model), false);
}

/**
* Sets the provided value to the provided path in the document.
* Any non existing path element will be created. If the last element is a list,
Expand Down

0 comments on commit f9a6ec1

Please sign in to comment.