Skip to content

Commit

Permalink
ingest: date_index_name processor template resolution (#31841)
Browse files Browse the repository at this point in the history 
This change adds support for template snippet (e.g. {{foo}}) resolution
in the date_index_name processor. The following configuration options
will now resolve a templated value if so configured:

* index_name_prefix  (e.g "index_name_prefix": "myindex-{{foo}}-")
* date_rounding (e.g. "date_rounding" : "{{bar}}")
* index_name_format (e.g."index_name_format": "{{baz}}")
  • Loading branch information
@jakelandis
jakelandis committed Jul 11, 2018
1 parent 9b7439b commit f19d83f
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 34 deletions.
47 changes: 33 additions & 14 deletions ...s/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
Expand All @@ -42,21 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
public static final String TYPE = "date_index_name";

private final String field;
private final String indexNamePrefix;
private final String dateRounding;
private final String indexNameFormat;
private final TemplateScript.Factory indexNamePrefixTemplate;
private final TemplateScript.Factory dateRoundingTemplate;
private final TemplateScript.Factory indexNameFormatTemplate;
private final DateTimeZone timezone;
private final List<Function<String, DateTime>> dateFormats;

DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
String indexNamePrefix, String dateRounding, String indexNameFormat) {
TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
TemplateScript.Factory indexNameFormatTemplate) {
super(tag);
this.field = field;
this.timezone = timezone;
this.dateFormats = dateFormats;
this.indexNamePrefix = indexNamePrefix;
this.dateRounding = dateRounding;
this.indexNameFormat = indexNameFormat;
this.indexNamePrefixTemplate = indexNamePrefixTemplate;
this.dateRoundingTemplate = dateRoundingTemplate;
this.indexNameFormatTemplate = indexNameFormatTemplate;
}

@Override
Expand All @@ -83,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception {
if (dateTime == null) {
throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
}
String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate);
String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate);
String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate);

DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
StringBuilder builder = new StringBuilder()
Expand All @@ -106,16 +112,16 @@ String getField() {
return field;
}

String getIndexNamePrefix() {
return indexNamePrefix;
TemplateScript.Factory getIndexNamePrefixTemplate() {
return indexNamePrefixTemplate;
}

String getDateRounding() {
return dateRounding;
TemplateScript.Factory getDateRoundingTemplate() {
return dateRoundingTemplate;
}

String getIndexNameFormat() {
return indexNameFormat;
TemplateScript.Factory getIndexNameFormatTemplate() {
return indexNameFormatTemplate;
}

DateTimeZone getTimezone() {
Expand All @@ -128,6 +134,12 @@ List<Function<String, DateTime>> getDateFormats() {

public static final class Factory implements Processor.Factory {

private final ScriptService scriptService;

public Factory(ScriptService scriptService) {
this.scriptService = scriptService;
}

@Override
public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, String tag,
Map<String, Object> config) throws Exception {
Expand All @@ -154,9 +166,16 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St

String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
TemplateScript.Factory indexNamePrefixTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService);
String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
TemplateScript.Factory dateRoundingTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService);
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat);
TemplateScript.Factory indexNameFormatTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate,
dateRoundingTemplate, indexNameFormatTemplate);
}
}

Expand Down
6 changes: 3 additions & 3 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory());
processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService));
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService));
processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters)));
processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
Expand All @@ -97,12 +97,12 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
Supplier<DiscoveryNodes> nodesInCluster) {
return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
}

@Override
public List<Setting<?>> getSettings() {
return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME);
}

private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) {
long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis();
long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis();
Expand Down
19 changes: 10 additions & 9 deletions ...ngest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,35 @@
package org.elasticsearch.ingest.common;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import org.joda.time.DateTimeZone;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

public class DateIndexNameFactoryTests extends ESTestCase {

public void testDefaults() throws Exception {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("date_rounding", "y");

DateIndexNameProcessor processor = factory.create(null, null, config);
assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
assertThat(processor.getField(), Matchers.equalTo("_field"));
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
assertThat(processor.getDateRounding(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
}

public void testSpecifyOptionalSettings() throws Exception {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("index_name_prefix", "_prefix");
Expand All @@ -63,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception {
config.put("index_name_format", "yyyyMMdd");

processor = factory.create(null, null, config);
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd"));

config = new HashMap<>();
config.put("field", "_field");
Expand All @@ -80,11 +82,11 @@ public void testSpecifyOptionalSettings() throws Exception {
config.put("date_rounding", "y");

processor = factory.create(null, null, config);
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix"));
}

public void testRequiredFields() throws Exception {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("date_rounding", "y");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
Expand All @@ -95,5 +97,4 @@ public void testRequiredFields() throws Exception {
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
}

}
45 changes: 37 additions & 8 deletions ...est-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,14 @@
package org.elasticsearch.ingest.common;

import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;

import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;

Expand All @@ -33,11 +36,8 @@ public class DateIndexNameProcessorTests extends ESTestCase {

public void testJodaPattern() throws Exception {
Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
DateIndexNameProcessor processor = new DateIndexNameProcessor(
"_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
"events-", "y", "yyyyMMdd"
);

DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z"));
processor.execute(document);
Expand All @@ -46,7 +46,7 @@ public void testJodaPattern() throws Exception {

public void testTAI64N()throws Exception {
Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
Expand All @@ -56,7 +56,7 @@ public void testTAI64N()throws Exception {

public void testUnixMs()throws Exception {
Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
Collections.singletonMap("_field", "1000500"));
Expand All @@ -71,12 +71,41 @@ public void testUnixMs()throws Exception {

public void testUnix()throws Exception {
Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
Collections.singletonMap("_field", "1000.5"));
dateProcessor.execute(document);
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
}

public void testTemplatedFields() throws Exception {
String indexNamePrefix = randomAlphaOfLength(10);
String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s");
String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy");
String date = Integer.toString(randomInt());
Function<String, DateTime> dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);

DateIndexNameProcessor dateProcessor = createProcessor("_field",
Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix,
dateRounding, indexNameFormat);

IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
Collections.singletonMap("_field", date));
dateProcessor.execute(document);

assertThat(document.getSourceAndMetadata().get("_index"),
equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat)
.print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>"));
}

private DateIndexNameProcessor createProcessor(String field, List<Function<String, DateTime>> dateFormats,
DateTimeZone timezone, String indexNamePrefix, String dateRounding,
String indexNameFormat) {
return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone,
new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix),
new TestTemplateService.MockTemplateScript.Factory(dateRounding),
new TestTemplateService.MockTemplateScript.Factory(indexNameFormat)
);
}
}

0 comments on commit f19d83f

Please sign in to comment.