From c21907bae45ddcb4edd065ad383dc24886f428b5 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 9 Jul 2018 10:24:34 +0200 Subject: [PATCH] Ingest: Add ignore_missing option to RemoveProc (#31693) Added `ignore_missing` setting to the RemoveProcessor to fix #23086 --- docs/reference/ingest/ingest-node.asciidoc | 5 +++-- .../ingest/common/RemoveProcessor.java | 18 ++++++++++++++--- .../ingest/common/RemoveProcessorTests.java | 20 ++++++++++++++++--- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 0872f00abfee8..aa72c1e868a96 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1767,8 +1767,9 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown .Remove Options [options="header"] |====== -| Name | Required | Default | Description -| `field` | yes | - | Fields to be removed +| Name | Required | Default | Description +| `field` | yes | - | Fields to be removed +| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document |====== Here is an example to remove a single field: diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java index 32d213694b1b5..61dc97ee0ef3c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java @@ -39,10 +39,12 @@ public final class RemoveProcessor extends AbstractProcessor { public static final String TYPE = "remove"; private final List fields; + private final boolean ignoreMissing; - RemoveProcessor(String tag, List fields) { + RemoveProcessor(String tag, List fields, boolean ignoreMissing) { super(tag); this.fields = new ArrayList<>(fields); + this.ignoreMissing = ignoreMissing; } public List getFields() { @@ -51,7 +53,16 @@ public List getFields() { @Override public void execute(IngestDocument document) { - fields.forEach(document::removeField); + if (ignoreMissing) { + fields.forEach(field -> { + String path = document.renderTemplate(field); + if (document.hasField(path)) { + document.removeField(path); + } + }); + } else { + fields.forEach(document::removeField); + } } @Override @@ -81,7 +92,8 @@ public RemoveProcessor create(Map registry, String pr final List compiledTemplates = fields.stream() .map(f -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", f, scriptService)) .collect(Collectors.toList()); - return new RemoveProcessor(processorTag, compiledTemplates); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + return new RemoveProcessor(processorTag, compiledTemplates, ignoreMissing); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java index 641bac8393977..b420b2fe3b00b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -37,7 +38,7 @@ public void testRemoveFields() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); Processor processor = new RemoveProcessor(randomAlphaOfLength(10), - Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field))); + Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)), false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(field), equalTo(false)); } @@ -45,8 +46,10 @@ public void testRemoveFields() throws Exception { public void testRemoveNonExistingField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new RemoveProcessor(randomAlphaOfLength(10), - Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(fieldName))); + Map config = new HashMap<>(); + config.put("field", fieldName); + String processorTag = randomAlphaOfLength(10); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config); try { processor.execute(ingestDocument); fail("remove field should have failed"); @@ -54,4 +57,15 @@ public void testRemoveNonExistingField() throws Exception { assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]")); } } + + public void testIgnoreMissing() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + String fieldName = RandomDocumentPicks.randomFieldName(random()); + Map config = new HashMap<>(); + config.put("field", fieldName); + config.put("ignore_missing", true); + String processorTag = randomAlphaOfLength(10); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config); + processor.execute(ingestDocument); + } }