From 0e33bdb34b7a151946b9a162e1c5bbc96fe9afe9 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 19 Jul 2018 13:41:07 -0700 Subject: [PATCH 1/5] Ingest: Support integer and long hex values in convert This commit adds checks for hex formatted strings in the convert processor, allowing strings like `0x1` to be parsed as integer `1`. closes #32182 --- .../ingest/common/ConvertProcessor.java | 12 ++++++++++-- .../ingest/common/ConvertProcessorTests.java | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 264df6f4c5f24..599fd28e6a658 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -42,7 +42,11 @@ enum Type { @Override public Object convert(Object value) { try { - return Integer.parseInt(value.toString()); + String str = value.toString(); + if (str.startsWith("0x")) { + return Integer.decode(str); + } + return Integer.parseInt(str); } catch(NumberFormatException e) { throw new IllegalArgumentException("unable to convert [" + value + "] to integer", e); } @@ -52,7 +56,11 @@ public Object convert(Object value) { @Override public Object convert(Object value) { try { - return Long.parseLong(value.toString()); + String str = value.toString(); + if (str.startsWith("0x")) { + return Long.decode(str); + } + return Long.parseLong(str); } catch(NumberFormatException e) { throw new IllegalArgumentException("unable to convert [" + value + "] to long", e); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 292a03d7d9033..982e8714cf632 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -49,6 +49,15 @@ public void testConvertInt() throws Exception { assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); } + public void testConvertIntHex() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + int randomInt = randomInt(); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "0x" + Integer.toHexString(randomInt)); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); + } + public void testConvertIntList() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int numItems = randomIntBetween(1, 10); @@ -92,6 +101,15 @@ public void testConvertLong() throws Exception { assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); } + public void testConvertLongHex() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + long randomLong = randomLong(); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "0x" + Long.toHexString(randomLong)); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); + } + public void testConvertLongList() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int numItems = randomIntBetween(1, 10); From fb51b2d6228d2bb19a361a7cff82d452dce2bd42 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 19 Jul 2018 14:21:02 -0700 Subject: [PATCH 2/5] switch to always use decode, and make tests handle negatives --- .../ingest/common/ConvertProcessor.java | 12 ++---------- .../ingest/common/ConvertProcessorTests.java | 6 ++++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 599fd28e6a658..ea594f6a15052 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -42,11 +42,7 @@ enum Type { @Override public Object convert(Object value) { try { - String str = value.toString(); - if (str.startsWith("0x")) { - return Integer.decode(str); - } - return Integer.parseInt(str); + return Integer.decode(value.toString()); } catch(NumberFormatException e) { throw new IllegalArgumentException("unable to convert [" + value + "] to integer", e); } @@ -56,11 +52,7 @@ public Object convert(Object value) { @Override public Object convert(Object value) { try { - String str = value.toString(); - if (str.startsWith("0x")) { - return Long.decode(str); - } - return Long.parseLong(str); + return Long.decode(value.toString()); } catch(NumberFormatException e) { throw new IllegalArgumentException("unable to convert [" + value + "] to long", e); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 982e8714cf632..4a6ce21b2dc51 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -52,7 +52,8 @@ public void testConvertInt() throws Exception { public void testConvertIntHex() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int randomInt = randomInt(); - String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "0x" + Integer.toHexString(randomInt)); + String intString = randomInt < 0 ? "-0x" + Integer.toHexString(-randomInt) : "0x" + Integer.toHexString(randomInt); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, intString); Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); @@ -104,7 +105,8 @@ public void testConvertLong() throws Exception { public void testConvertLongHex() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); long randomLong = randomLong(); - String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "0x" + Long.toHexString(randomLong)); + String longString = randomLong < 0 ? "-0x" + Long.toHexString(-randomLong) : "0x" + Long.toHexString(randomLong); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, longString); Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); From c406980bd4a7eb99a93df9653ad833324b629cf0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 23 Jul 2018 14:30:03 -0700 Subject: [PATCH 3/5] switch back to decode only for 0x prefix --- .../ingest/common/ConvertProcessor.java | 12 ++++++++++-- .../ingest/common/ConvertProcessorTests.java | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index ea594f6a15052..7fe852cf698de 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -42,7 +42,11 @@ enum Type { @Override public Object convert(Object value) { try { - return Integer.decode(value.toString()); + String strValue = value.toString(); + if (strValue.startsWith("0x")) { + return Integer.decode(strValue); + } + return Integer.parseInt(strValue); } catch(NumberFormatException e) { throw new IllegalArgumentException("unable to convert [" + value + "] to integer", e); } @@ -52,7 +56,11 @@ public Object convert(Object value) { @Override public Object convert(Object value) { try { - return Long.decode(value.toString()); + String strValue = value.toString(); + if (strValue.startsWith("0x")) { + return Long.decode(strValue); + } + return Long.parseLong(strValue); } catch(NumberFormatException e) { throw new IllegalArgumentException("unable to convert [" + value + "] to long", e); } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 4a6ce21b2dc51..c8bf2c519ed41 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -59,6 +59,15 @@ public void testConvertIntHex() throws Exception { assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); } + public void testConvertIntHexError() { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + String value = "0x" + randomAlphaOfLengthBetween(1, 10); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); + assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to integer")); + } + public void testConvertIntList() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int numItems = randomIntBetween(1, 10); @@ -112,6 +121,15 @@ public void testConvertLongHex() throws Exception { assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); } + public void testConvertLongHexError() { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + String value = "0x" + randomAlphaOfLengthBetween(1, 10); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); + assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to long")); + } + public void testConvertLongList() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int numItems = randomIntBetween(1, 10); From 54c30ca9f25d2ef725ce66842fb07e331c1a522c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Jul 2018 08:53:26 -0700 Subject: [PATCH 4/5] fix negatives and add octal test --- .../ingest/common/ConvertProcessor.java | 4 ++-- .../ingest/common/ConvertProcessorTests.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 7fe852cf698de..2e881b82b59de 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -43,7 +43,7 @@ enum Type { public Object convert(Object value) { try { String strValue = value.toString(); - if (strValue.startsWith("0x")) { + if (strValue.startsWith("0x") || strValue.startsWith("-0x")) { return Integer.decode(strValue); } return Integer.parseInt(strValue); @@ -57,7 +57,7 @@ public Object convert(Object value) { public Object convert(Object value) { try { String strValue = value.toString(); - if (strValue.startsWith("0x")) { + if (strValue.startsWith("0x") || strValue.startsWith("-0x")) { return Long.decode(strValue); } return Long.parseLong(strValue); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index c8bf2c519ed41..8d6f62cb781c3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -59,6 +59,14 @@ public void testConvertIntHex() throws Exception { assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); } + public void testConvertIntLeadingZero() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010"); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10)); + } + public void testConvertIntHexError() { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String value = "0x" + randomAlphaOfLengthBetween(1, 10); @@ -121,6 +129,14 @@ public void testConvertLongHex() throws Exception { assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong)); } + public void testConvertLongLeadingZero() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010"); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10)); + } + public void testConvertLongHexError() { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String value = "0x" + randomAlphaOfLengthBetween(1, 10); From 9d37495eefcb3a1e1e3717b90c9d371ae9a06d93 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 24 Jul 2018 09:10:02 -0700 Subject: [PATCH 5/5] fix type --- .../org/elasticsearch/ingest/common/ConvertProcessorTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 8d6f62cb781c3..f0fc31dab3533 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -134,7 +134,7 @@ public void testConvertLongLeadingZero() throws Exception { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010"); Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false); processor.execute(ingestDocument); - assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10)); + assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L)); } public void testConvertLongHexError() {