diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index b44eaa3bfa3ca..0d6253c88f9fa 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -32,6 +32,8 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -42,21 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor { public static final String TYPE = "date_index_name"; private final String field; - private final String indexNamePrefix; - private final String dateRounding; - private final String indexNameFormat; + private final TemplateScript.Factory indexNamePrefixTemplate; + private final TemplateScript.Factory dateRoundingTemplate; + private final TemplateScript.Factory indexNameFormatTemplate; private final DateTimeZone timezone; private final List> dateFormats; DateIndexNameProcessor(String tag, String field, List> dateFormats, DateTimeZone timezone, - String indexNamePrefix, String dateRounding, String indexNameFormat) { + TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate, + TemplateScript.Factory indexNameFormatTemplate) { super(tag); this.field = field; this.timezone = timezone; this.dateFormats = dateFormats; - this.indexNamePrefix = indexNamePrefix; - this.dateRounding = dateRounding; - this.indexNameFormat = indexNameFormat; + this.indexNamePrefixTemplate = indexNamePrefixTemplate; + this.dateRoundingTemplate = dateRoundingTemplate; + this.indexNameFormatTemplate = indexNameFormatTemplate; } @Override @@ -83,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception { if (dateTime == null) { throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException); } + String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate); + String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate); + String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate); DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat); StringBuilder builder = new StringBuilder() @@ -106,16 +112,16 @@ String getField() { return field; } - String getIndexNamePrefix() { - return indexNamePrefix; + TemplateScript.Factory getIndexNamePrefixTemplate() { + return indexNamePrefixTemplate; } - String getDateRounding() { - return dateRounding; + TemplateScript.Factory getDateRoundingTemplate() { + return dateRoundingTemplate; } - String getIndexNameFormat() { - return indexNameFormat; + TemplateScript.Factory getIndexNameFormatTemplate() { + return indexNameFormatTemplate; } DateTimeZone getTimezone() { @@ -128,6 +134,12 @@ List> getDateFormats() { public static final class Factory implements Processor.Factory { + private final ScriptService scriptService; + + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + @Override public DateIndexNameProcessor create(Map registry, String tag, Map config) throws Exception { @@ -154,9 +166,16 @@ public DateIndexNameProcessor create(Map registry, St String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", ""); + TemplateScript.Factory indexNamePrefixTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService); String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding"); + TemplateScript.Factory dateRoundingTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService); String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd"); - return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat); + TemplateScript.Factory indexNameFormatTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService); + return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate, + dateRoundingTemplate, indexNameFormatTemplate); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index d9878cae9e2ab..bc475a2a00539 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -73,7 +73,7 @@ public Map getProcessors(Processor.Parameters paramet processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory()); processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService)); processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory()); - processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory()); + processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService)); processors.put(SortProcessor.TYPE, new SortProcessor.Factory()); processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters))); processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService)); @@ -97,12 +97,12 @@ public List getRestHandlers(Settings settings, RestController restC Supplier nodesInCluster) { return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); } - + @Override public List> getSettings() { return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME); } - + private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) { long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis(); long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis(); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 3b9e2121c9511..2735cf55776b0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -20,18 +20,20 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; import org.joda.time.DateTimeZone; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; public class DateIndexNameFactoryTests extends ESTestCase { public void testDefaults() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("field", "_field"); config.put("date_rounding", "y"); @@ -39,14 +41,14 @@ public void testDefaults() throws Exception { DateIndexNameProcessor processor = factory.create(null, null, config); assertThat(processor.getDateFormats().size(), Matchers.equalTo(1)); assertThat(processor.getField(), Matchers.equalTo("_field")); - assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("")); - assertThat(processor.getDateRounding(), Matchers.equalTo("y")); - assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd")); + assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("")); + assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y")); + assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd")); assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC)); } public void testSpecifyOptionalSettings() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("field", "_field"); config.put("index_name_prefix", "_prefix"); @@ -63,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("index_name_format", "yyyyMMdd"); processor = factory.create(null, null, config); - assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd")); + assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd")); config = new HashMap<>(); config.put("field", "_field"); @@ -80,11 +82,11 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); processor = factory.create(null, null, config); - assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix")); + assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix")); } public void testRequiredFields() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("date_rounding", "y"); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); @@ -95,5 +97,4 @@ public void testRequiredFields() throws Exception { e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing")); } - } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index eba37dc742169..c97da116e3489 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -19,11 +19,14 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -33,11 +36,8 @@ public class DateIndexNameProcessorTests extends ESTestCase { public void testJodaPattern() throws Exception { Function function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT); - DateIndexNameProcessor processor = new DateIndexNameProcessor( - "_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC, - "events-", "y", "yyyyMMdd" - ); - + DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function), + DateTimeZone.UTC, "events-", "y", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z")); processor.execute(document); @@ -46,7 +46,7 @@ public void testJodaPattern() throws Exception { public void testTAI64N()throws Exception { Function function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024")); @@ -56,7 +56,7 @@ public void testTAI64N()throws Exception { public void testUnixMs()throws Exception { Function function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "1000500")); @@ -71,7 +71,7 @@ public void testUnixMs()throws Exception { public void testUnix()throws Exception { Function function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "1000.5")); @@ -79,4 +79,33 @@ public void testUnix()throws Exception { assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } + public void testTemplatedFields() throws Exception { + String indexNamePrefix = randomAlphaOfLength(10); + String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy"); + String date = Integer.toString(randomInt()); + Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); + + DateIndexNameProcessor dateProcessor = createProcessor("_field", + Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix, + dateRounding, indexNameFormat); + + IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, + Collections.singletonMap("_field", date)); + dateProcessor.execute(document); + + assertThat(document.getSourceAndMetadata().get("_index"), + equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat) + .print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>")); + } + + private DateIndexNameProcessor createProcessor(String field, List> dateFormats, + DateTimeZone timezone, String indexNamePrefix, String dateRounding, + String indexNameFormat) { + return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone, + new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix), + new TestTemplateService.MockTemplateScript.Factory(dateRounding), + new TestTemplateService.MockTemplateScript.Factory(indexNameFormat) + ); + } }