From efd9ad936aeea96c3eb3aa86abaa777ac894bb09 Mon Sep 17 00:00:00 2001 From: uklimaschewski Date: Sun, 19 Mar 2023 13:40:32 +0100 Subject: [PATCH 1/3] 360: BUG: Undefined operator '.' ParseException when letter 'e' is after dot '.' --- .../com/ezylang/evalex/parser/Tokenizer.java | 65 ++++++++++++------- .../ExpressionEvaluatorScientificTest.java | 5 +- .../ExpressionEvaluatorStructureTest.java | 29 +++++++++ .../evalex/parser/TokenizerStructureTest.java | 45 +++++++++++++ 4 files changed, 121 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java index 989f15eb..3a6a23bb 100644 --- a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java +++ b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java @@ -103,9 +103,9 @@ private boolean implicitMultiplicationPossible(Token currentToken) { } return ((previousToken.getType() == BRACE_CLOSE && currentToken.getType() == BRACE_OPEN) - || ((previousToken.getType() == NUMBER_LITERAL - && currentToken.getType() == VARIABLE_OR_CONSTANT)) - || ((previousToken.getType() == NUMBER_LITERAL && currentToken.getType() == BRACE_OPEN))); + || (previousToken.getType() == NUMBER_LITERAL + && currentToken.getType() == VARIABLE_OR_CONSTANT) + || (previousToken.getType() == NUMBER_LITERAL && currentToken.getType() == BRACE_OPEN)); } private void validateToken(Token currentToken) throws ParseException { @@ -150,16 +150,16 @@ private Token getNextToken() throws ParseException { } else if (currentChar == ']' && configuration.isArraysAllowed()) { return parseArrayClose(); } else if (currentChar == '.' - && !isNumberChar(peekNextChar()) + && !isNextCharNumberChar() && configuration.isStructuresAllowed()) { return parseStructureSeparator(); } else if (currentChar == ',') { Token token = new Token(currentColumnIndex, ",", TokenType.COMMA); consumeChar(); return token; - } else if (isIdentifierStart(currentChar)) { + } else if (isAtIdentifierStart()) { return parseIdentifier(); - } else if (isNumberStart(currentChar)) { + } else if (isAtNumberStart()) { return parseNumberLiteral(); } else { return parseOperator(); @@ -247,6 +247,8 @@ private Token parseOperator() throws ParseException { } else if (operatorDictionary.hasInfixOperator(tokenString)) { OperatorIfc operator = operatorDictionary.getInfixOperator(tokenString); return new Token(tokenStartIndex, tokenString, TokenType.INFIX_OPERATOR, operator); + } else if (tokenString.equals(".") && configuration.isStructuresAllowed()) { + return new Token(tokenStartIndex, tokenString, STRUCTURE_SEPARATOR); } throw new ParseException( tokenStartIndex, @@ -357,14 +359,14 @@ private Token parseNumberLiteral() throws ParseException { consumeChar(); tokenValue.append((char) currentChar); consumeChar(); - while (currentChar != -1 && isHexChar(currentChar)) { + while (currentChar != -1 && isAtHexChar()) { tokenValue.append((char) currentChar); consumeChar(); } } else { // decimal number int lastChar = -1; - while (currentChar != -1 && isNumberChar(currentChar)) { + while (currentChar != -1 && isAtNumberChar()) { tokenValue.append((char) currentChar); lastChar = currentChar; consumeChar(); @@ -382,7 +384,7 @@ private Token parseNumberLiteral() throws ParseException { private Token parseIdentifier() throws ParseException { int tokenStartIndex = currentColumnIndex; StringBuilder tokenValue = new StringBuilder(); - while (currentChar != -1 && isIdentifierChar(currentChar)) { + while (currentChar != -1 && isAtIdentifierChar()) { tokenValue.append((char) currentChar); consumeChar(); } @@ -472,24 +474,43 @@ private char escapeCharacter(int character) throws ParseException { } } - private boolean isNumberStart(int ch) { - if (Character.isDigit(ch)) { + private boolean isAtNumberStart() { + if (Character.isDigit(currentChar)) { return true; } - return ch == '.' && Character.isDigit(peekNextChar()); + return currentChar == '.' && Character.isDigit(peekNextChar()); } - private boolean isNumberChar(int ch) { + private boolean isAtNumberChar() { int previousChar = peekPreviousChar(); + if (previousChar == 'e' || previousChar == 'E') { - return Character.isDigit(ch) || ch == '+' || ch == '-'; - } else { - return Character.isDigit(ch) || ch == '.' || ch == 'e' || ch == 'E'; + return Character.isDigit(currentChar) || currentChar == '+' || currentChar == '-'; + } + + if (previousChar == '.') { + return Character.isDigit(currentChar) || currentChar == 'e' || currentChar == 'E'; + } + + return Character.isDigit(currentChar) + || currentChar == '.' + || currentChar == 'e' + || currentChar == 'E'; + } + + private boolean isNextCharNumberChar() { + if (peekNextChar() == -1) { + return false; } + consumeChar(); + boolean isAtNumber = isAtNumberChar(); + currentColumnIndex--; + currentChar = expressionString.charAt(currentColumnIndex - 1); + return isAtNumber; } - private boolean isHexChar(int ch) { - switch (ch) { + private boolean isAtHexChar() { + switch (currentChar) { case '0': case '1': case '2': @@ -518,12 +539,12 @@ private boolean isHexChar(int ch) { } } - private boolean isIdentifierStart(int ch) { - return Character.isLetter(ch) || ch == '_'; + private boolean isAtIdentifierStart() { + return Character.isLetter(currentChar) || currentChar == '_'; } - private boolean isIdentifierChar(int ch) { - return Character.isLetter(ch) || Character.isDigit(ch) || ch == '_'; + private boolean isAtIdentifierChar() { + return Character.isLetter(currentChar) || Character.isDigit(currentChar) || currentChar == '_'; } private void skipBlanks() { diff --git a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java index de06e967..aadcc591 100644 --- a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java +++ b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java @@ -37,7 +37,10 @@ class ExpressionEvaluatorScientificTest extends BaseExpressionEvaluatorTest { "0.5e2 : 50", "0.35E4 + 0.5e1 : 3505", "2135E-4 : 0.2135", - "2135E+4 : 21350000" + "2135E+4 : 21350000", + "3.e1: 30", + "3.e-1: 0.3", + "3.e+2: 300" }) void testScientificLiteralsEvaluation(String expression, String expectedResult) throws ParseException, EvaluationException { diff --git a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java index d981a32f..94209845 100644 --- a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java +++ b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorStructureTest.java @@ -26,6 +26,35 @@ class ExpressionEvaluatorStructureTest extends BaseExpressionEvaluatorTest { + @Test + void testStructureScientificNumberDistinction() throws EvaluationException, ParseException { + Map structure = + new HashMap<>() { + { + put("environment_id", new BigDecimal(12345)); + } + }; + Expression expression = new Expression("order.environment_id").with("order", structure); + + assertThat(expression.evaluate().getStringValue()).isEqualTo("12345"); + } + + @Test + void testStructureScientificNumberDistinctionMultiple() + throws EvaluationException, ParseException { + Map structure1 = new HashMap<>(); + Map structure2 = new HashMap<>(); + Map structure3 = new HashMap<>(); + + structure3.put("e", new BigDecimal("765")); + structure2.put("var_x", structure3); + structure1.put("e_id_e", structure2); + + Expression expression = new Expression("order.e_id_e.var_x.e").with("order", structure1); + + assertThat(expression.evaluate().getStringValue()).isEqualTo("765"); + } + @Test void testSimpleStructure() throws ParseException, EvaluationException { Map structure = diff --git a/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java b/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java index ea63232c..a44ce40a 100644 --- a/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java +++ b/src/test/java/com/ezylang/evalex/parser/TokenizerStructureTest.java @@ -32,6 +32,51 @@ void testStructureSimple() throws ParseException { new Token(3, "b", TokenType.VARIABLE_OR_CONSTANT)); } + @Test + void testStructureLeftIsE() throws ParseException { + assertAllTokensParsedCorrectly( + "e.b", + new Token(1, "e", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "b", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureRightIsE() throws ParseException { + assertAllTokensParsedCorrectly( + "a.e", + new Token(1, "a", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "e", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureBothAreE() throws ParseException { + assertAllTokensParsedCorrectly( + "e.e", + new Token(1, "e", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "e", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureLeftEndsE() throws ParseException { + assertAllTokensParsedCorrectly( + "variable.a", + new Token(1, "variable", TokenType.VARIABLE_OR_CONSTANT), + new Token(9, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(10, "a", TokenType.VARIABLE_OR_CONSTANT)); + } + + @Test + void testStructureRightStartsE() throws ParseException { + assertAllTokensParsedCorrectly( + "a.end", + new Token(1, "a", TokenType.VARIABLE_OR_CONSTANT), + new Token(2, ".", TokenType.STRUCTURE_SEPARATOR), + new Token(3, "end", TokenType.VARIABLE_OR_CONSTANT)); + } + @Test void testStructureSeparatorNotAllowedBegin() { assertThatThrownBy(() -> new Tokenizer(".", configuration).parse()) From d2a988196f55a1fdab54f78fbffe483f66dce2f5 Mon Sep 17 00:00:00 2001 From: uklimaschewski Date: Sun, 19 Mar 2023 13:45:50 +0100 Subject: [PATCH 2/3] 360: Raises test coverage --- .../evalex/ExpressionEvaluatorScientificTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java index aadcc591..6e44010a 100644 --- a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java +++ b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java @@ -15,12 +15,12 @@ */ package com.ezylang.evalex; -import static org.assertj.core.api.Assertions.assertThat; - import com.ezylang.evalex.parser.ParseException; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import static org.assertj.core.api.Assertions.assertThat; + class ExpressionEvaluatorScientificTest extends BaseExpressionEvaluatorTest { @ParameterizedTest @@ -40,7 +40,10 @@ class ExpressionEvaluatorScientificTest extends BaseExpressionEvaluatorTest { "2135E+4 : 21350000", "3.e1: 30", "3.e-1: 0.3", - "3.e+2: 300" + "3.e+2: 300", + "3.E1: 30", + "3.E-1: 0.3", + "3.E+2: 300" }) void testScientificLiteralsEvaluation(String expression, String expectedResult) throws ParseException, EvaluationException { From 6816e863d0c2bc925182c1e9cbe51e1d0b951ba7 Mon Sep 17 00:00:00 2001 From: uklimaschewski Date: Sun, 19 Mar 2023 13:47:52 +0100 Subject: [PATCH 3/3] 360: Fixes formatting --- .../com/ezylang/evalex/ExpressionEvaluatorScientificTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java index 6e44010a..af6f8f63 100644 --- a/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java +++ b/src/test/java/com/ezylang/evalex/ExpressionEvaluatorScientificTest.java @@ -15,12 +15,12 @@ */ package com.ezylang.evalex; +import static org.assertj.core.api.Assertions.assertThat; + import com.ezylang.evalex.parser.ParseException; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import static org.assertj.core.api.Assertions.assertThat; - class ExpressionEvaluatorScientificTest extends BaseExpressionEvaluatorTest { @ParameterizedTest