From 43047f81107c7594ca8a8f720c6b6ea689aea06b Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 18 Jul 2018 14:23:25 +0200 Subject: [PATCH 1/6] INGEST: Make a few Processors callable by Painless * Extracted a few stateless String processors as well as the json processor to static methods and whitelisted them in Painless --- .../ingest/common/BytesProcessor.java | 4 +- .../ingest/common/JsonProcessor.java | 46 +------- .../ingest/common/LowercaseProcessor.java | 5 +- .../ingest/common/TrimProcessor.java | 4 +- .../ingest/common/URLDecodeProcessor.java | 10 +- .../ingest/common/UppercaseProcessor.java | 5 +- .../ingest/common/BytesProcessorTests.java | 4 +- .../painless/spi/org.elasticsearch.txt | 10 ++ .../org/elasticsearch/ingest/Processors.java | 100 ++++++++++++++++++ 9 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/ingest/Processors.java diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java index dfe9a054acf0..1e0da37233ec 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java @@ -19,7 +19,7 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.ingest.Processors; import java.util.Map; @@ -37,7 +37,7 @@ public final class BytesProcessor extends AbstractStringProcessor { @Override protected Long process(String value) { - return ByteSizeValue.parseBytesSizeValue(value, null, getField()).getBytes(); + return Processors.bytes(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index 2f217735df2f..5ce60ab35bae 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -19,19 +19,12 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.Processors; -import java.io.IOException; -import java.io.InputStream; import java.util.Map; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -69,39 +62,10 @@ boolean isAddToRoot() { @Override public void execute(IngestDocument document) throws Exception { - Object fieldValue = document.getFieldValue(field, Object.class); - BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString()); - try (InputStream stream = bytesRef.streamInput(); - XContentParser parser = JsonXContent.jsonXContent - .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) { - XContentParser.Token token = parser.nextToken(); - Object value = null; - if (token == XContentParser.Token.VALUE_NULL) { - value = null; - } else if (token == XContentParser.Token.VALUE_STRING) { - value = parser.text(); - } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = parser.numberValue(); - } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - value = parser.booleanValue(); - } else if (token == XContentParser.Token.START_OBJECT) { - value = parser.map(); - } else if (token == XContentParser.Token.START_ARRAY) { - value = parser.list(); - } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - throw new IllegalArgumentException("cannot read binary value"); - } - if (addToRoot && (value instanceof Map)) { - for (Map.Entry entry : ((Map) value).entrySet()) { - document.setFieldValue(entry.getKey(), entry.getValue()); - } - } else if (addToRoot) { - throw new IllegalArgumentException("cannot add non-map fields to root of document"); - } else { - document.setFieldValue(targetField, value); - } - } catch (IOException e) { - throw new IllegalArgumentException(e); + if (addToRoot) { + Processors.json(document.getSourceAndMetadata(), field); + } else { + document.setFieldValue(targetField, Processors.json(document.getFieldValue(field, Object.class))); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index aef8b0cce24f..3bf8f90b9a5c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -19,7 +19,8 @@ package org.elasticsearch.ingest.common; -import java.util.Locale; +import org.elasticsearch.ingest.Processors; + import java.util.Map; /** @@ -37,7 +38,7 @@ public final class LowercaseProcessor extends AbstractStringProcessor { @Override protected String process(String value) { - return value.toLowerCase(Locale.ROOT); + return Processors.lowercase(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java index 98fe1223e539..4677fc271bd9 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java @@ -19,6 +19,8 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.ingest.Processors; + import java.util.Map; /** @@ -35,7 +37,7 @@ public final class TrimProcessor extends AbstractStringProcessor { @Override protected String process(String value) { - return value.trim(); + return Processors.trim(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java index 945419499ad4..df273b6eeaba 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java @@ -19,8 +19,8 @@ package org.elasticsearch.ingest.common; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; +import org.elasticsearch.ingest.Processors; + import java.util.Map; /** @@ -36,11 +36,7 @@ public final class URLDecodeProcessor extends AbstractStringProcessor { @Override protected String process(String value) { - try { - return URLDecoder.decode(value, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("could not URL-decode field[" + getField() + "]", e); - } + return Processors.urlDecode(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java index af93f06a8f27..22215d2cb8b0 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java @@ -19,7 +19,8 @@ package org.elasticsearch.ingest.common; -import java.util.Locale; +import org.elasticsearch.ingest.Processors; + import java.util.Map; /** @@ -36,7 +37,7 @@ public final class UppercaseProcessor extends AbstractStringProcessor { @Override protected String process(String value) { - return value.toUpperCase(Locale.ROOT); + return Processors.uppercase(value); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java index 0da3434adf17..788340a455a4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java @@ -63,7 +63,7 @@ public void testTooLarge() { Processor processor = newProcessor(fieldName, randomBoolean(), fieldName); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument)); assertThat(exception.getMessage(), - CoreMatchers.equalTo("failed to parse setting [" + fieldName + "] with value [8912pb] as a size in bytes")); + CoreMatchers.equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes")); assertThat(exception.getCause().getMessage(), CoreMatchers.containsString("Values greater than 9223372036854775807 bytes are not supported")); } @@ -93,6 +93,6 @@ public void testFractional() throws Exception { processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L)); assertWarnings("Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " + - "[" + fieldName + "]"); + "[Ingest Field]"); } } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt index 6495659d9cdc..a22c754844a5 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt @@ -175,3 +175,13 @@ class org.elasticsearch.index.similarity.ScriptedSimilarity$Doc { int getLength() float getFreq() } + +class org.elasticsearch.ingest.Processors { + long bytes(String) + String lowercase(String) + String uppercase(String) + Object json(Object) + void json(Map, String) + String trim(String) + String urlDecode(String) +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/ingest/Processors.java b/server/src/main/java/org/elasticsearch/ingest/Processors.java new file mode 100644 index 000000000000..0241d635b33c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/Processors.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.Locale; +import java.util.Map; + +public final class Processors { + + public static long bytes(String value) { + return ByteSizeValue.parseBytesSizeValue(value, null, "Ingest Field").getBytes(); + } + + public static String lowercase(String value) { + return value.toLowerCase(Locale.ROOT); + } + + public static String uppercase(String value) { + return value.toUpperCase(Locale.ROOT); + } + + public static String trim(String value) { + return value.trim(); + } + + public static Object json(Object fieldValue) { + BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString()); + try (InputStream stream = bytesRef.streamInput(); + XContentParser parser = JsonXContent.jsonXContent + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) { + XContentParser.Token token = parser.nextToken(); + Object value = null; + if (token == XContentParser.Token.VALUE_NULL) { + value = null; + } else if (token == XContentParser.Token.VALUE_STRING) { + value = parser.text(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.numberValue(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + value = parser.booleanValue(); + } else if (token == XContentParser.Token.START_OBJECT) { + value = parser.map(); + } else if (token == XContentParser.Token.START_ARRAY) { + value = parser.list(); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + throw new IllegalArgumentException("cannot read binary value"); + } + return value; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + @SuppressWarnings("unchecked") + public static void json(Map ctx, String field) { + Object value = json(ctx.get(field)); + if (value instanceof Map) { + ctx.putAll((Map) value); + } else { + throw new IllegalArgumentException("cannot add non-map fields to root of document"); + } + } + + public static String urlDecode(String value) { + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Could not URL-decode value.", e); + } + } +} From 3fec863e9888621144f55586bded8a9a905fc019 Mon Sep 17 00:00:00 2001 From: Armin Date: Thu, 19 Jul 2018 11:35:34 +0200 Subject: [PATCH 2/6] provide whitelist from processors plugin --- modules/ingest-common/build.gradle | 2 + .../ingest/common/BytesProcessor.java | 2 - .../ingest/common/JsonProcessor.java | 1 - .../ingest/common/LowercaseProcessor.java | 2 - .../ingest/common}/Processors.java | 2 +- .../common/ProcessorsWhitelistExtension.java | 41 +++++++++++++++++++ .../ingest/common/TrimProcessor.java | 2 - .../ingest/common/URLDecodeProcessor.java | 2 - .../ingest/common/UppercaseProcessor.java | 2 - ...asticsearch.painless.spi.PainlessExtension | 1 + .../ingest/common/processors_whitelist.txt | 30 ++++++++++++++ .../painless/spi/org.elasticsearch.txt | 10 ----- 12 files changed, 75 insertions(+), 22 deletions(-) rename {server/src/main/java/org/elasticsearch/ingest => modules/ingest-common/src/main/java/org/elasticsearch/ingest/common}/Processors.java (98%) create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java create mode 100644 modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 424c1197da3e..58f2803ee1fd 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -20,9 +20,11 @@ esplugin { description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' classname 'org.elasticsearch.ingest.common.IngestCommonPlugin' + extendedPlugins = ['lang-painless'] } dependencies { + compileOnly project(':modules:lang-painless') compile project(':libs:grok') } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java index 1e0da37233ec..63ae62b84711 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.ingest.Processors; - import java.util.Map; /** diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index 5ce60ab35bae..cbbbddc66e02 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -23,7 +23,6 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; -import org.elasticsearch.ingest.Processors; import java.util.Map; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index 3bf8f90b9a5c..71c0b1c311d9 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.ingest.Processors; - import java.util.Map; /** diff --git a/server/src/main/java/org/elasticsearch/ingest/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java similarity index 98% rename from server/src/main/java/org/elasticsearch/ingest/Processors.java rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java index 0241d635b33c..406b5d97d299 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Processors.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java new file mode 100644 index 000000000000..ced84057c7a9 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.IngestScript; +import org.elasticsearch.script.ScriptContext; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ProcessorsWhitelistExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = + WhitelistLoader.loadFromResourceFiles(ProcessorsWhitelistExtension.class, "processors_whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + return Collections.singletonMap(IngestScript.CONTEXT, Collections.singletonList(WHITELIST)); + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java index 4677fc271bd9..b528399f9b23 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.ingest.Processors; - import java.util.Map; /** diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java index df273b6eeaba..d38f09a33aec 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.ingest.Processors; - import java.util.Map; /** diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java index 22215d2cb8b0..4f3714256a9f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.ingest.Processors; - import java.util.Map; /** diff --git a/modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 000000000000..8a98f034be56 --- /dev/null +++ b/modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.ingest.common.ProcessorsWhitelistExtension \ No newline at end of file diff --git a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt new file mode 100644 index 000000000000..1f54f68ade61 --- /dev/null +++ b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt @@ -0,0 +1,30 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This file contains a whitelist of static processor methods that can be accessed from painless + +class org.elasticsearch.ingest.common.Processors { + long bytes(String) + String lowercase(String) + String uppercase(String) + Object json(Object) + void json(Map, String) + String trim(String) + String urlDecode(String) +} \ No newline at end of file diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt index a22c754844a5..8491d15c27eb 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt @@ -174,14 +174,4 @@ class org.elasticsearch.index.similarity.ScriptedSimilarity$Term { class org.elasticsearch.index.similarity.ScriptedSimilarity$Doc { int getLength() float getFreq() -} - -class org.elasticsearch.ingest.Processors { - long bytes(String) - String lowercase(String) - String uppercase(String) - Object json(Object) - void json(Map, String) - String trim(String) - String urlDecode(String) } \ No newline at end of file From 0f14e7492aa627b0598af3248213919eefaa12e8 Mon Sep 17 00:00:00 2001 From: Armin Date: Thu, 19 Jul 2018 14:24:03 +0200 Subject: [PATCH 3/6] provide whitelist from processors plugin --- .../ingest/common/BytesProcessor.java | 8 ++- .../ingest/common/JsonProcessor.java | 51 ++++++++++++++- .../ingest/common/LowercaseProcessor.java | 7 ++- .../ingest/common/Processors.java | 63 ++----------------- .../ingest/common/TrimProcessor.java | 2 +- .../ingest/common/URLDecodeProcessor.java | 12 +++- .../ingest/common/UppercaseProcessor.java | 7 ++- .../ingest/common/processors_whitelist.txt | 1 - 8 files changed, 86 insertions(+), 65 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java index 63ae62b84711..d07b56e1b3df 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java @@ -19,6 +19,8 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.unit.ByteSizeValue; + import java.util.Map; /** @@ -33,9 +35,13 @@ public final class BytesProcessor extends AbstractStringProcessor { super(processorTag, field, ignoreMissing, targetField); } + public static long apply(String value) { + return ByteSizeValue.parseBytesSizeValue(value, null, "Ingest Field").getBytes(); + } + @Override protected Long process(String value) { - return Processors.bytes(value); + return apply(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index cbbbddc66e02..c0a9d37abdab 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -19,11 +19,19 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import java.io.IOException; +import java.io.InputStream; import java.util.Map; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -59,12 +67,51 @@ boolean isAddToRoot() { return addToRoot; } + public static Object apply(Object fieldValue) { + BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString()); + try (InputStream stream = bytesRef.streamInput(); + XContentParser parser = JsonXContent.jsonXContent + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) { + XContentParser.Token token = parser.nextToken(); + Object value = null; + if (token == XContentParser.Token.VALUE_NULL) { + value = null; + } else if (token == XContentParser.Token.VALUE_STRING) { + value = parser.text(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.numberValue(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + value = parser.booleanValue(); + } else if (token == XContentParser.Token.START_OBJECT) { + value = parser.map(); + } else if (token == XContentParser.Token.START_ARRAY) { + value = parser.list(); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + throw new IllegalArgumentException("cannot read binary value"); + } + return value; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + public static void apply(Map ctx, String fieldName) { + Object value = apply(ctx.get(fieldName)); + if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) value; + ctx.putAll(map); + } else { + throw new IllegalArgumentException("cannot add non-map fields to root of document"); + } + } + @Override public void execute(IngestDocument document) throws Exception { if (addToRoot) { - Processors.json(document.getSourceAndMetadata(), field); + apply(document.getSourceAndMetadata(), field); } else { - document.setFieldValue(targetField, Processors.json(document.getFieldValue(field, Object.class))); + document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class))); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index 71c0b1c311d9..4269cb05257f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import java.util.Locale; import java.util.Map; /** @@ -34,9 +35,13 @@ public final class LowercaseProcessor extends AbstractStringProcessor { super(processorTag, field, ignoreMissing, targetField); } + public static String apply(String value) { + return value.toLowerCase(Locale.ROOT); + } + @Override protected String process(String value) { - return Processors.lowercase(value); + return apply(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java index 406b5d97d299..8a0b15298924 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java @@ -19,82 +19,31 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.Locale; import java.util.Map; public final class Processors { public static long bytes(String value) { - return ByteSizeValue.parseBytesSizeValue(value, null, "Ingest Field").getBytes(); + return BytesProcessor.apply(value); } public static String lowercase(String value) { - return value.toLowerCase(Locale.ROOT); + return LowercaseProcessor.apply(value); } public static String uppercase(String value) { - return value.toUpperCase(Locale.ROOT); - } - - public static String trim(String value) { - return value.trim(); + return UppercaseProcessor.apply(value); } public static Object json(Object fieldValue) { - BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString()); - try (InputStream stream = bytesRef.streamInput(); - XContentParser parser = JsonXContent.jsonXContent - .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) { - XContentParser.Token token = parser.nextToken(); - Object value = null; - if (token == XContentParser.Token.VALUE_NULL) { - value = null; - } else if (token == XContentParser.Token.VALUE_STRING) { - value = parser.text(); - } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = parser.numberValue(); - } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - value = parser.booleanValue(); - } else if (token == XContentParser.Token.START_OBJECT) { - value = parser.map(); - } else if (token == XContentParser.Token.START_ARRAY) { - value = parser.list(); - } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - throw new IllegalArgumentException("cannot read binary value"); - } - return value; - } catch (IOException e) { - throw new IllegalArgumentException(e); - } + return JsonProcessor.apply(fieldValue); } - @SuppressWarnings("unchecked") public static void json(Map ctx, String field) { - Object value = json(ctx.get(field)); - if (value instanceof Map) { - ctx.putAll((Map) value); - } else { - throw new IllegalArgumentException("cannot add non-map fields to root of document"); - } + JsonProcessor.apply(ctx, field); } public static String urlDecode(String value) { - try { - return URLDecoder.decode(value, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("Could not URL-decode value.", e); - } + return URLDecodeProcessor.apply(value); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java index b528399f9b23..98fe1223e539 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java @@ -35,7 +35,7 @@ public final class TrimProcessor extends AbstractStringProcessor { @Override protected String process(String value) { - return Processors.trim(value); + return value.trim(); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java index d38f09a33aec..fb6c5acf98b2 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java @@ -19,6 +19,8 @@ package org.elasticsearch.ingest.common; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; import java.util.Map; /** @@ -32,9 +34,17 @@ public final class URLDecodeProcessor extends AbstractStringProcessor { super(processorTag, field, ignoreMissing, targetField); } + public static String apply(String value) { + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Could not URL-decode value.", e); + } + } + @Override protected String process(String value) { - return Processors.urlDecode(value); + return apply(value); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java index 4f3714256a9f..6c428627c7d7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import java.util.Locale; import java.util.Map; /** @@ -33,9 +34,13 @@ public final class UppercaseProcessor extends AbstractStringProcessor { super(processorTag, field, ignoreMissing, targetField); } + public static String apply(String value) { + return value.toUpperCase(Locale.ROOT); + } + @Override protected String process(String value) { - return Processors.uppercase(value); + return apply(value); } @Override diff --git a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt index 1f54f68ade61..3d93b19f0660 100644 --- a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt +++ b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt @@ -25,6 +25,5 @@ class org.elasticsearch.ingest.common.Processors { String uppercase(String) Object json(Object) void json(Map, String) - String trim(String) String urlDecode(String) } \ No newline at end of file From c8216cfc98724c6e658a43f3eea757e2f012b372 Mon Sep 17 00:00:00 2001 From: Armin Date: Thu, 19 Jul 2018 14:55:13 +0200 Subject: [PATCH 4/6] provide whitelist from processors plugin --- modules/ingest-common/build.gradle | 4 ++ .../test/ingest/190_script_processor.yml | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 58f2803ee1fd..4f35bbee28df 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -30,3 +30,7 @@ dependencies { compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" + +integTestCluster { + module project(':modules:lang-painless') +} \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml new file mode 100644 index 000000000000..f14c4b75beda --- /dev/null +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -0,0 +1,42 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + +--- +"Test bytes processor": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "ctx.bytes_target_field = Processors.bytes(ctx.bytes_source_field)" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {bytes_source_field: "1kb"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.bytes_source_field: "1kb" } + - match: { _source.bytes_target_field: 1024 } + From 6ebef1cb37188a3680ba914d5897b5ffc8823f2b Mon Sep 17 00:00:00 2001 From: Armin Date: Thu, 19 Jul 2018 15:21:50 +0200 Subject: [PATCH 5/6] provide whitelist from processors plugin --- .../test/ingest/190_script_processor.yml | 149 +++++++++++++++++- 1 file changed, 144 insertions(+), 5 deletions(-) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml index f14c4b75beda..1261b07b77da 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -6,7 +6,7 @@ teardown: ignore: 404 --- -"Test bytes processor": +"Test invoke bytes processor": - do: ingest.put_pipeline: id: "my_pipeline" @@ -17,7 +17,7 @@ teardown: { "script" : { "lang": "painless", - "source" : "ctx.bytes_target_field = Processors.bytes(ctx.bytes_source_field)" + "source" : "ctx.target_field = Processors.bytes(ctx.source_field)" } } ] @@ -30,13 +30,152 @@ teardown: type: test id: 1 pipeline: "my_pipeline" - body: {bytes_source_field: "1kb"} + body: {source_field: "1kb"} - do: get: index: test type: test id: 1 - - match: { _source.bytes_source_field: "1kb" } - - match: { _source.bytes_target_field: 1024 } + - match: { _source.source_field: "1kb" } + - match: { _source.target_field: 1024 } +--- +"Test invoke lowercase processor": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "ctx.target_field = Processors.lowercase(ctx.source_field)" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {source_field: "FooBar"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.source_field: "FooBar" } + - match: { _source.target_field: "foobar" } + +--- +"Test invoke uppercase processor": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "ctx.target_field = Processors.uppercase(ctx.source_field)" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {source_field: "FooBar"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.source_field: "FooBar" } + - match: { _source.target_field: "FOOBAR" } + +--- +"Test invoke json processor, assign to field": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "ctx.target_field = Processors.json(ctx.source_field)" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {source_field: "{\"foo\":\"bar\"}"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.source_field: "{\"foo\":\"bar\"}" } + - match: { _source.target_field.foo: "bar" } + +--- +"Test invoke json processor, assign to root": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "Processors.json(ctx, 'source_field')" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {source_field: "{\"foo\":\"bar\"}"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.source_field: "{\"foo\":\"bar\"}" } + - match: { _source.foo: "bar" } From f58d5971758b6a216c0028af5c67d1db00a1ab9f Mon Sep 17 00:00:00 2001 From: Armin Date: Thu, 19 Jul 2018 15:26:47 +0200 Subject: [PATCH 6/6] provide whitelist from processors plugin --- .../test/ingest/190_script_processor.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml index 1261b07b77da..bd55b764a95a 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -179,3 +179,38 @@ teardown: id: 1 - match: { _source.source_field: "{\"foo\":\"bar\"}" } - match: { _source.foo: "bar" } + +--- +"Test invoke urlDecode processor": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "ctx.target_field = Processors.urlDecode(ctx.source_field)" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {source_field: "foo%20bar"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.source_field: "foo%20bar" } + - match: { _source.target_field: "foo bar" }