adds a token validation for infix operators to tokenizer (#348)

ezylang · Feb 12, 2023 · 4073767 · 4073767
1 parent 568e8ac
commit 4073767
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 67 deletions.
diff --git a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java
@@ -16,6 +16,7 @@
 package com.ezylang.evalex.parser;
 
 import static com.ezylang.evalex.parser.Token.TokenType.BRACE_OPEN;
+import static com.ezylang.evalex.parser.Token.TokenType.INFIX_OPERATOR;
 
 import com.ezylang.evalex.config.ExpressionConfiguration;
 import com.ezylang.evalex.config.FunctionDictionaryIfc;
@@ -75,6 +76,7 @@ public List<Token> parse() throws ParseException {
           throw new ParseException(currentToken, "Missing operator");
         }
       }
+      validateToken(currentToken);
       tokens.add(currentToken);
       currentToken = getNextToken();
     }
@@ -90,6 +92,26 @@ public List<Token> parse() throws ParseException {
     return tokens;
   }
 
+  private void validateToken(Token currentToken) throws ParseException {
+    Token previousToken = getPreviousToken();
+    if (previousToken != null
+        && previousToken.getType() == INFIX_OPERATOR
+        && invalidTokenAfterInfixOperator(currentToken)) {
+      throw new ParseException(currentToken, "Unexpected token after infix operator");
+    }
+  }
+
+  private boolean invalidTokenAfterInfixOperator(Token token) {
+    switch (token.getType()) {
+      case INFIX_OPERATOR:
+      case BRACE_CLOSE:
+      case COMMA:
+        return true;
+      default:
+        return false;
+    }
+  }
+
   private Token getNextToken() throws ParseException {
 
     // blanks are always skipped.

diff --git a/src/test/java/com/ezylang/evalex/parser/ShuntingYardExceptionsTest.java b/src/test/java/com/ezylang/evalex/parser/ShuntingYardExceptionsTest.java
@@ -15,12 +15,7 @@
 */
 package com.ezylang.evalex.parser;
 
-import static com.ezylang.evalex.parser.Token.TokenType.FUNCTION_PARAM_START;
-import static com.ezylang.evalex.parser.Token.TokenType.INFIX_OPERATOR;
-import static com.ezylang.evalex.parser.Token.TokenType.POSTFIX_OPERATOR;
-import static com.ezylang.evalex.parser.Token.TokenType.PREFIX_OPERATOR;
-import static com.ezylang.evalex.parser.Token.TokenType.STRUCTURE_SEPARATOR;
-import static com.ezylang.evalex.parser.Token.TokenType.VARIABLE_OR_CONSTANT;
+import static com.ezylang.evalex.parser.Token.TokenType.*;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import com.ezylang.evalex.Expression;
@@ -29,6 +24,9 @@
 import java.util.Arrays;
 import java.util.List;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 class ShuntingYardExceptionsTest extends BaseParserTest {
 
@@ -143,75 +141,37 @@ void testFunctionTooManyParameters() {
         .hasMessage("Too many parameters for function");
   }
 
-  @Test
-  void testTooManyOperands() {
-    Expression expression = new Expression("1 2");
-
-    assertThatThrownBy(expression::evaluate)
-        .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
-  }
-
-  @Test
-  void testTooManyOperandsString() {
-    Expression expression = new Expression("Hello World");
+  @ParameterizedTest
+  @ValueSource(
+      strings = {
+        "Hello, World",
+        "Hello ROUND(1,2) + (1 + 1)",
+        "Hello ROUND(1,2)",
+        "Hello 1 + (1 + 1)",
+        "Hello 1 + 1",
+        "Hello World",
+        "Hello 1",
+        "1 2"
+      })
+  void testTooManyOperands(String expressionString) {
+    Expression expression = new Expression(expressionString);
 
     assertThatThrownBy(expression::evaluate)
         .isInstanceOf(ParseException.class)
         .hasMessage("Too many operands");
   }
 
-  @Test
-  void testTooManyOperandsStringWithNumbers() {
-    Expression expression = new Expression("Hello 1");
-
-    assertThatThrownBy(expression::evaluate)
-        .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
-  }
-
-  @Test
-  void testTooManyOperandsStringWithNumbersAndOperators() {
-    Expression expression = new Expression("Hello 1 + 1");
-
-    assertThatThrownBy(expression::evaluate)
-        .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
-  }
-
-  @Test
-  void testTooManyOperandsStringWithNumbersAndOperatorsAndBraces() {
-    Expression expression = new Expression("Hello 1 + (1 + 1)");
+  @ParameterizedTest
+  @CsvSource(
+      delimiter = ':',
+      value = {"(x+y)*(a-) : 10", "a** : 3", "5+, : 3"})
+  void testInvalidTokenAfterInfixOperator(String expressionString, int position) {
+    Expression expression = new Expression(expressionString);
 
     assertThatThrownBy(expression::evaluate)
         .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
-  }
-
-  @Test
-  void testTooManyOperandsStringWithFunctions() {
-    Expression expression = new Expression("Hello ROUND(1,2)");
-
-    assertThatThrownBy(expression::evaluate)
-        .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
-  }
-
-  @Test
-  void testTooManyOperandsStringWithFunctionsAndBraces() {
-    Expression expression = new Expression("Hello ROUND(1,2) + (1 + 1)");
-
-    assertThatThrownBy(expression::evaluate)
-        .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
-  }
-
-  @Test
-  void testTooManyOperandsStringWithSpecialCharacters() {
-    Expression expression = new Expression("Hello, World");
-
-    assertThatThrownBy(expression::evaluate)
-        .isInstanceOf(ParseException.class)
-        .hasMessage("Too many operands");
+        .hasMessage("Unexpected token after infix operator")
+        .extracting("startPosition")
+        .isEqualTo(position);
   }
 }