From 86d6b1b0415b9ebbfddaf17be4c7ee4b5d1a7d21 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Thu, 5 Jul 2018 14:59:39 -0500 Subject: [PATCH 1/2] ingest: date_index_name processor template resolution This change adds support for template snippet (e.g. {{foo}}) resolution in the date_index_name processor. The following configuration options will now resolve a templated value if so configured: * index_name_prefix (e.g "index_name_prefix": "myindex-{{foo}}-") * date_rounding (e.g. "date_rounding" : "{{bar}}") --- .../ingest/common/DateIndexNameProcessor.java | 17 +++++++-- .../ingest/common/IngestCommonPlugin.java | 6 ++-- .../common/DateIndexNameFactoryTests.java | 8 ++--- .../common/DateIndexNameProcessorTests.java | 36 ++++++++++++++++--- .../elasticsearch/ingest/IngestDocument.java | 15 ++++++++ 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index b44eaa3bfa3ca..11f61997f8c6a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -32,6 +32,8 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.ValueSource; +import org.elasticsearch.script.ScriptService; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -47,9 +49,10 @@ public final class DateIndexNameProcessor extends AbstractProcessor { private final String indexNameFormat; private final DateTimeZone timezone; private final List> dateFormats; + private final ScriptService scriptService; DateIndexNameProcessor(String tag, String field, List> dateFormats, DateTimeZone timezone, - String indexNamePrefix, String dateRounding, String indexNameFormat) { + String indexNamePrefix, String dateRounding, String indexNameFormat, ScriptService scriptService) { super(tag); this.field = field; this.timezone = timezone; @@ -57,6 +60,7 @@ public final class DateIndexNameProcessor extends AbstractProcessor { this.indexNamePrefix = indexNamePrefix; this.dateRounding = dateRounding; this.indexNameFormat = indexNameFormat; + this.scriptService = scriptService; } @Override @@ -94,7 +98,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { .append('}') .append('>'); String dynamicIndexName = builder.toString(); - ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName); + ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), ValueSource.wrap(dynamicIndexName, scriptService)); } @Override @@ -128,6 +132,12 @@ List> getDateFormats() { public static final class Factory implements Processor.Factory { + private final ScriptService scriptService; + + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + @Override public DateIndexNameProcessor create(Map registry, String tag, Map config) throws Exception { @@ -156,7 +166,8 @@ public DateIndexNameProcessor create(Map registry, St String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", ""); String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding"); String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd"); - return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat); + return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, + dateRounding, indexNameFormat, scriptService); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index bc900d325104a..c8809cbce81f4 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -73,7 +73,7 @@ public Map getProcessors(Processor.Parameters paramet processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory()); processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService)); processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory()); - processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory()); + processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService)); processors.put(SortProcessor.TYPE, new SortProcessor.Factory()); processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters))); processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService)); @@ -97,12 +97,12 @@ public List getRestHandlers(Settings settings, RestController restC Supplier nodesInCluster) { return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); } - + @Override public List> getSettings() { return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME); } - + private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) { long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis(); long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis(); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 3b9e2121c9511..34e90c8747378 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; import org.joda.time.DateTimeZone; @@ -31,7 +32,7 @@ public class DateIndexNameFactoryTests extends ESTestCase { public void testDefaults() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("field", "_field"); config.put("date_rounding", "y"); @@ -46,7 +47,7 @@ public void testDefaults() throws Exception { } public void testSpecifyOptionalSettings() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("field", "_field"); config.put("index_name_prefix", "_prefix"); @@ -84,7 +85,7 @@ public void testSpecifyOptionalSettings() throws Exception { } public void testRequiredFields() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("date_rounding", "y"); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); @@ -95,5 +96,4 @@ public void testRequiredFields() throws Exception { e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing")); } - } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index eba37dc742169..58b4fdc8ecadf 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -19,15 +19,26 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.mockito.Matchers; +import org.mockito.Mockito; import java.util.Collections; import java.util.Locale; import java.util.function.Function; +import static org.elasticsearch.script.Script.DEFAULT_TEMPLATE_LANG; import static org.hamcrest.CoreMatchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class DateIndexNameProcessorTests extends ESTestCase { @@ -35,7 +46,7 @@ public void testJodaPattern() throws Exception { Function function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT); DateIndexNameProcessor processor = new DateIndexNameProcessor( "_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC, - "events-", "y", "yyyyMMdd" + "events-", "y", "yyyyMMdd", TestTemplateService.instance() ); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, @@ -47,7 +58,7 @@ public void testJodaPattern() throws Exception { public void testTAI64N()throws Exception { Function function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null); DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); + DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance()); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024")); dateProcessor.execute(document); @@ -57,7 +68,7 @@ public void testTAI64N()throws Exception { public void testUnixMs()throws Exception { Function function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null); DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); + DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance()); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "1000500")); dateProcessor.execute(document); @@ -72,11 +83,28 @@ public void testUnixMs()throws Exception { public void testUnix()throws Exception { Function function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); + DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance()); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "1000.5")); dateProcessor.execute(document); assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } + public void testTemplateSupported() throws Exception { + ScriptService scriptService = mock(ScriptService.class); + TestTemplateService.MockTemplateScript.Factory factory = new TestTemplateService.MockTemplateScript.Factory("script_result"); + when(scriptService.compile(any(Script.class), Matchers.>any())).thenReturn(factory); + when(scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG)).thenReturn(true); + + DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", + Collections.singletonList(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null)), + DateTimeZone.UTC, "events-", "m", "yyyyMMdd", scriptService); + IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, + Collections.singletonMap("_field", "1000.5")); + dateProcessor.execute(document); + + // here we only care that the script was compiled and that it returned what we expect. + Mockito.verify(scriptService).compile(any(Script.class), Matchers.>any()); + assertThat(document.getSourceAndMetadata().get("_index"), equalTo("script_result")); + } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 2bd842e72b107..95904f85026a9 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -406,6 +406,21 @@ public void setFieldValue(String path, Object value) { setFieldValue(path, value, false); } + /** + * Sets the provided value to the provided path in the document. + * Any non existing path element will be created. + * If the last item in the path is a list, the value will replace the existing list as a whole. + * Use {@link #appendFieldValue(String, Object)} to append values to lists instead. + * @param path The path within the document in dot-notation + * @param valueSource The value source that will produce the value or values to append to the existing ones + * @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the + * item identified by the provided path. + */ + public void setFieldValue(String path, ValueSource valueSource) { + Map model = valueSource == null ? null : createTemplateModel(); + setFieldValue(path, valueSource == null ? null : valueSource.copyAndResolve(model), false); + } + /** * Sets the provided value to the provided path in the document. * Any non existing path element will be created. If the last element is a list, From f9a6ec1e18c054b2184f03b6cfd429a940d75f1c Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 10 Jul 2018 12:48:18 -0500 Subject: [PATCH 2/2] ingest: date_index_name processor template resolution Review changes: * Change to TemplateScript.Factory instead of ValueSource.wrap * Add support for templated index_name_format * Remove unecessary changes --- .../ingest/common/DateIndexNameProcessor.java | 46 +++++++------ .../common/DateIndexNameFactoryTests.java | 11 +-- .../common/DateIndexNameProcessorTests.java | 67 ++++++++++--------- .../elasticsearch/ingest/IngestDocument.java | 15 ----- 4 files changed, 67 insertions(+), 72 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index 11f61997f8c6a..0d6253c88f9fa 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -32,8 +32,8 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; -import org.elasticsearch.ingest.ValueSource; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -44,23 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor { public static final String TYPE = "date_index_name"; private final String field; - private final String indexNamePrefix; - private final String dateRounding; - private final String indexNameFormat; + private final TemplateScript.Factory indexNamePrefixTemplate; + private final TemplateScript.Factory dateRoundingTemplate; + private final TemplateScript.Factory indexNameFormatTemplate; private final DateTimeZone timezone; private final List> dateFormats; - private final ScriptService scriptService; DateIndexNameProcessor(String tag, String field, List> dateFormats, DateTimeZone timezone, - String indexNamePrefix, String dateRounding, String indexNameFormat, ScriptService scriptService) { + TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate, + TemplateScript.Factory indexNameFormatTemplate) { super(tag); this.field = field; this.timezone = timezone; this.dateFormats = dateFormats; - this.indexNamePrefix = indexNamePrefix; - this.dateRounding = dateRounding; - this.indexNameFormat = indexNameFormat; - this.scriptService = scriptService; + this.indexNamePrefixTemplate = indexNamePrefixTemplate; + this.dateRoundingTemplate = dateRoundingTemplate; + this.indexNameFormatTemplate = indexNameFormatTemplate; } @Override @@ -87,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception { if (dateTime == null) { throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException); } + String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate); + String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate); + String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate); DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat); StringBuilder builder = new StringBuilder() @@ -98,7 +100,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { .append('}') .append('>'); String dynamicIndexName = builder.toString(); - ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), ValueSource.wrap(dynamicIndexName, scriptService)); + ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName); } @Override @@ -110,16 +112,16 @@ String getField() { return field; } - String getIndexNamePrefix() { - return indexNamePrefix; + TemplateScript.Factory getIndexNamePrefixTemplate() { + return indexNamePrefixTemplate; } - String getDateRounding() { - return dateRounding; + TemplateScript.Factory getDateRoundingTemplate() { + return dateRoundingTemplate; } - String getIndexNameFormat() { - return indexNameFormat; + TemplateScript.Factory getIndexNameFormatTemplate() { + return indexNameFormatTemplate; } DateTimeZone getTimezone() { @@ -164,10 +166,16 @@ public DateIndexNameProcessor create(Map registry, St String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", ""); + TemplateScript.Factory indexNamePrefixTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService); String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding"); + TemplateScript.Factory dateRoundingTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService); String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd"); - return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, - dateRounding, indexNameFormat, scriptService); + TemplateScript.Factory indexNameFormatTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService); + return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate, + dateRoundingTemplate, indexNameFormatTemplate); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 34e90c8747378..2735cf55776b0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -26,6 +26,7 @@ import org.joda.time.DateTimeZone; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -40,9 +41,9 @@ public void testDefaults() throws Exception { DateIndexNameProcessor processor = factory.create(null, null, config); assertThat(processor.getDateFormats().size(), Matchers.equalTo(1)); assertThat(processor.getField(), Matchers.equalTo("_field")); - assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("")); - assertThat(processor.getDateRounding(), Matchers.equalTo("y")); - assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd")); + assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("")); + assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y")); + assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd")); assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC)); } @@ -64,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("index_name_format", "yyyyMMdd"); processor = factory.create(null, null, config); - assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd")); + assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd")); config = new HashMap<>(); config.put("field", "_field"); @@ -81,7 +82,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); processor = factory.create(null, null, config); - assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix")); + assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix")); } public void testRequiredFields() throws Exception { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 58b4fdc8ecadf..c97da116e3489 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -20,35 +20,24 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.TestTemplateService; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.TemplateScript; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.mockito.Matchers; -import org.mockito.Mockito; +import org.joda.time.format.DateTimeFormat; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.function.Function; -import static org.elasticsearch.script.Script.DEFAULT_TEMPLATE_LANG; import static org.hamcrest.CoreMatchers.equalTo; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class DateIndexNameProcessorTests extends ESTestCase { public void testJodaPattern() throws Exception { Function function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT); - DateIndexNameProcessor processor = new DateIndexNameProcessor( - "_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC, - "events-", "y", "yyyyMMdd", TestTemplateService.instance() - ); - + DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function), + DateTimeZone.UTC, "events-", "y", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z")); processor.execute(document); @@ -57,8 +46,8 @@ public void testJodaPattern() throws Exception { public void testTAI64N()throws Exception { Function function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance()); + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), + DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024")); dateProcessor.execute(document); @@ -67,8 +56,8 @@ public void testTAI64N()throws Exception { public void testUnixMs()throws Exception { Function function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance()); + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), + DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "1000500")); dateProcessor.execute(document); @@ -82,29 +71,41 @@ public void testUnixMs()throws Exception { public void testUnix()throws Exception { Function function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance()); + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), + DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, Collections.singletonMap("_field", "1000.5")); dateProcessor.execute(document); assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } - public void testTemplateSupported() throws Exception { - ScriptService scriptService = mock(ScriptService.class); - TestTemplateService.MockTemplateScript.Factory factory = new TestTemplateService.MockTemplateScript.Factory("script_result"); - when(scriptService.compile(any(Script.class), Matchers.>any())).thenReturn(factory); - when(scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG)).thenReturn(true); + public void testTemplatedFields() throws Exception { + String indexNamePrefix = randomAlphaOfLength(10); + String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy"); + String date = Integer.toString(randomInt()); + Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); + + DateIndexNameProcessor dateProcessor = createProcessor("_field", + Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix, + dateRounding, indexNameFormat); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", - Collections.singletonList(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null)), - DateTimeZone.UTC, "events-", "m", "yyyyMMdd", scriptService); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, - Collections.singletonMap("_field", "1000.5")); + Collections.singletonMap("_field", date)); dateProcessor.execute(document); - // here we only care that the script was compiled and that it returned what we expect. - Mockito.verify(scriptService).compile(any(Script.class), Matchers.>any()); - assertThat(document.getSourceAndMetadata().get("_index"), equalTo("script_result")); + assertThat(document.getSourceAndMetadata().get("_index"), + equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat) + .print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>")); + } + + private DateIndexNameProcessor createProcessor(String field, List> dateFormats, + DateTimeZone timezone, String indexNamePrefix, String dateRounding, + String indexNameFormat) { + return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone, + new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix), + new TestTemplateService.MockTemplateScript.Factory(dateRounding), + new TestTemplateService.MockTemplateScript.Factory(indexNameFormat) + ); } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 95904f85026a9..2bd842e72b107 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -406,21 +406,6 @@ public void setFieldValue(String path, Object value) { setFieldValue(path, value, false); } - /** - * Sets the provided value to the provided path in the document. - * Any non existing path element will be created. - * If the last item in the path is a list, the value will replace the existing list as a whole. - * Use {@link #appendFieldValue(String, Object)} to append values to lists instead. - * @param path The path within the document in dot-notation - * @param valueSource The value source that will produce the value or values to append to the existing ones - * @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the - * item identified by the provided path. - */ - public void setFieldValue(String path, ValueSource valueSource) { - Map model = valueSource == null ? null : createTemplateModel(); - setFieldValue(path, valueSource == null ? null : valueSource.copyAndResolve(model), false); - } - /** * Sets the provided value to the provided path in the document. * Any non existing path element will be created. If the last element is a list,