Merge pull request #76 from markus-s24/artur

Power and truncated division operators

src/main/java/com/hubspot/jinjava/el/ext/ExtendedParser.java

@@ -52,9 +52,16 @@ public class ExtendedParser extends Parser {
   static final Scanner.ExtensionToken LITERAL_DICT_START = new Scanner.ExtensionToken("{");
   static final Scanner.ExtensionToken LITERAL_DICT_END = new Scanner.ExtensionToken("}");
 
+  static final Scanner.ExtensionToken TRUNC_DIV = new Scanner.ExtensionToken("//");
+  static final Scanner.ExtensionToken POWER_OF  = new Scanner.ExtensionToken("**");
+
   static {
     ExtendedScanner.addKeyToken(IF);
     ExtendedScanner.addKeyToken(ELSE);
+
+    ExtendedScanner.addKeyToken(TruncDivOperator.TOKEN);
+    ExtendedScanner.addKeyToken(PowerOfOperator.TOKEN);
+
     ExtendedScanner.addKeyToken(CollectionMembershipOperator.TOKEN);
   }
 
@@ -64,6 +71,8 @@ public ExtendedParser(Builder context, String input) {
     putExtensionHandler(AbsOperator.TOKEN, AbsOperator.HANDLER);
     putExtensionHandler(NamedParameterOperator.TOKEN, NamedParameterOperator.HANDLER);
     putExtensionHandler(StringConcatOperator.TOKEN, StringConcatOperator.HANDLER);
+    putExtensionHandler(TruncDivOperator.TOKEN, TruncDivOperator.HANDLER);
+    putExtensionHandler(PowerOfOperator.TOKEN, PowerOfOperator.HANDLER);
 
     putExtensionHandler(CollectionMembershipOperator.TOKEN, CollectionMembershipOperator.HANDLER);
 
@@ -369,8 +378,14 @@ protected AstNode value() throws ScanException, ParseException {
 
           AstProperty exptestProperty = createAstDot(identifier(EXPTEST_PREFIX + exptestName), "evaluate", true);
           v = createAstMethod(exptestProperty, new AstParameters(exptestParams));
-        }
+        } else if ("//".equals(getToken().getImage()) && lookahead(0).getSymbol() == IDENTIFIER) {
+            consumeToken(); // '//'
+            v = createAstBinary(v, mul(true), TruncDivOperator.OP);
 
+        } else if ("**".equals(getToken().getImage()) && lookahead(0).getSymbol() == IDENTIFIER) {
+            consumeToken(); // '**'
+            v = createAstBinary(v, mul(true), PowerOfOperator.OP);
+        }
         return v;
       }
     }

src/main/java/com/hubspot/jinjava/el/ext/ExtendedScanner.java

@@ -115,6 +115,12 @@ protected Token nextEval() throws ScanException {
     char c1 = getInput().charAt(getPosition());
     char c2 = getPosition() < getInput().length() - 1 ? getInput().charAt(getPosition() + 1) : (char) 0;
 
+    if (c1 == '/' && c2 == '/') {
+        return ExtendedParser.TRUNC_DIV;
+    }
+    if (c1 == '*' && c2 == '*') {
+        return ExtendedParser.POWER_OF;
+    }
     if (c1 == '|' && c2 != '|') {
       return ExtendedParser.PIPE;
     }

src/main/java/com/hubspot/jinjava/el/ext/PowerOfOperator.java

@@ -0,0 +1,47 @@
+package com.hubspot.jinjava.el.ext;
+
+import de.odysseus.el.misc.TypeConverter;
+import de.odysseus.el.tree.impl.Parser.ExtensionHandler;
+import de.odysseus.el.tree.impl.Parser.ExtensionPoint;
+import de.odysseus.el.tree.impl.Scanner;
+import de.odysseus.el.tree.impl.ast.AstBinary;
+import de.odysseus.el.tree.impl.ast.AstBinary.SimpleOperator;
+import de.odysseus.el.tree.impl.ast.AstNode;
+
+public class PowerOfOperator extends SimpleOperator {
+
+  @Override
+  protected Object apply(TypeConverter converter, Object a, Object b) {
+      boolean aInt = a instanceof Integer || a instanceof Long;
+      boolean bInt = b instanceof Integer || b instanceof Long;
+      boolean aNum = aInt || a instanceof Double || a instanceof Float;
+      boolean bNum = bInt || b instanceof Double || b instanceof Float;
+
+      if (aInt && bInt) {
+          Long d = converter.convert(a, Long.class);
+          Long e = converter.convert(b, Long.class);
+          return (long)Math.pow(d, e);
+   }
+      if (aNum && bNum) {
+          Double d = converter.convert(a, Double.class);
+          Double e = converter.convert(b, Double.class);
+          return Math.pow(d, e);
+      }
+      throw new IllegalArgumentException("Unsupported operand type(s) for **: "
+              + "'" + a.getClass().getSimpleName() + "' and "
+              + "'" + b.getClass().getSimpleName() + "'");
+  }
+
+  public static final Scanner.ExtensionToken TOKEN = new Scanner.ExtensionToken("**");
+  public static final PowerOfOperator OP = new PowerOfOperator();
+
+  public static final ExtensionHandler HANDLER = new ExtensionHandler(ExtensionPoint.MUL) {
+    @Override
+    public AstNode createAstNode(AstNode... children) {
+      return new AstBinary(children[0], children[1], OP);
+    }
+  };
+
+}
+
+

src/main/java/com/hubspot/jinjava/el/ext/TruncDivOperator.java

@@ -0,0 +1,48 @@
+package com.hubspot.jinjava.el.ext;
+
+import de.odysseus.el.misc.TypeConverter;
+import de.odysseus.el.tree.impl.Parser.ExtensionHandler;
+import de.odysseus.el.tree.impl.Parser.ExtensionPoint;
+import de.odysseus.el.tree.impl.Scanner;
+import de.odysseus.el.tree.impl.ast.AstBinary;
+import de.odysseus.el.tree.impl.ast.AstBinary.SimpleOperator;
+import de.odysseus.el.tree.impl.ast.AstNode;
+
+public class TruncDivOperator extends SimpleOperator {
+
+  @Override
+  protected Object apply(TypeConverter converter, Object a, Object b) {
+
+      boolean aInt = a instanceof Integer || a instanceof Long;
+      boolean bInt = b instanceof Integer || b instanceof Long;
+      boolean aNum = aInt || a instanceof Double || a instanceof Float;
+      boolean bNum = bInt || b instanceof Double || b instanceof Float;
+
+      if (aInt && bInt) {
+          Long d = converter.convert(a, Long.class);
+          Long e = converter.convert(b, Long.class);
+          return Math.floorDiv(d, e);
+      }
+      if (aNum && bNum) {
+          Double d = converter.convert(a, Double.class);
+          Double e = converter.convert(b, Double.class);
+          return Math.floor(d/e);
+      }
+      throw new IllegalArgumentException("Unsupported operand type(s) for //: "
+              + "'" + a.getClass().getSimpleName() + "' and "
+              + "'" + b.getClass().getSimpleName() + "'");
+  }
+
+  public static final Scanner.ExtensionToken TOKEN = new Scanner.ExtensionToken("//");
+  public static final TruncDivOperator OP = new TruncDivOperator();
+
+  public static final ExtensionHandler HANDLER = new ExtensionHandler(ExtensionPoint.MUL) {
+    @Override
+    public AstNode createAstNode(AstNode... children) {
+      return new AstBinary(children[0], children[1], OP);
+    }
+  };
+
+}
+
+

src/main/java/com/hubspot/jinjava/lib/tag/ForTag.java

@@ -71,7 +71,25 @@ public class ForTag implements Tag {
   @SuppressWarnings("unchecked")
   @Override
   public String interpret(TagNode tagNode, JinjavaInterpreter interpreter) {
-    List<String> helper = new HelperStringTokenizer(tagNode.getHelpers()).splitComma(true).allTokens();
+
+    /* apdlv72@gmail.com
+     * Fix for issues with for-loops that contain whitespace in their range, e.g.
+     * "{% for i in range(1 * 1, 2 * 2) %}"
+     * This is because HelperStringTokenizer will split the range expressions also
+     * at white spaces and end up with [i, in, range(1, *, 1, 2, *, 2)].
+     * To avoid this, the below fix will remove white space from the expression
+     * on the right side of the keyword "in". It will do so however only if there
+     * are no characters in this expression that indicate strings - namely ' and ".
+     * This avoids messing up expressions like {% for i in ['a ','b'] %} that
+     * contain spaces in the arguments.
+     * TODO A somewhat more sophisticated tokenizing/parsing of the for-loop expression.
+     */
+    String helpers = tagNode.getHelpers();
+    String parts[] = helpers.split("\\s+in\\s+");
+    if (2==parts.length && !parts[1].contains("'") && !parts[1].contains("\"") ) {
+        helpers = parts[0] + " in " + parts[1].replace(" ", "");
+    }
+    List<String> helper = new HelperStringTokenizer(helpers).splitComma(true).allTokens();
 
     List<String> loopVars = Lists.newArrayList();
     int inPos = 0;

src/test/java/com/hubspot/jinjava/el/ext/PowerOfTest.java

@@ -0,0 +1,63 @@
+package com.hubspot.jinjava.el.ext;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.hubspot.jinjava.Jinjava;
+import com.hubspot.jinjava.interpret.FatalTemplateErrorsException;
+
+public class PowerOfTest {
+
+    @Before
+    public void setUp() {
+        jinja = new Jinjava();
+    }
+
+    @Test
+    public void testPowerOfInteger() {
+
+        Map<String, Object> context = Maps.newHashMap();
+        context.put("base",     2);
+        context.put("exponent", 8);
+
+        String template = "{% set x = base ** exponent %}{{x}}";
+        String rendered = jinja.render(template, context);
+        assertEquals("256", rendered);
+    }
+
+    @Test
+    public void testPowerOfFractional() {
+
+        Map<String, Object> context = Maps.newHashMap();
+        context.put("base",     2);
+        context.put("exponent", 8.0);
+
+        String template = "{% set x = base ** exponent %}{{x}}";
+        String rendered = jinja.render(template, context);
+        assertEquals("256.0", rendered);
+    }
+
+    @Test
+    public void test04PowerOfStringFails() {
+
+        Map<String, Object> context = Maps.newHashMap();
+        context.put("base",     "2");
+        context.put("exponent", "8");
+
+        String template = "{% set x = base ** exponent %}{{x}}";
+        try {
+            jinja.render(template, context);
+        } catch (FatalTemplateErrorsException e) {
+            String msg = e.getMessage();
+            assertTrue(msg.contains("Unsupported operand type(s)"));
+        }
+    }
+
+    private Jinjava jinja;
+}

src/test/java/com/hubspot/jinjava/el/ext/TruncDivTest.java

@@ -0,0 +1,72 @@
+package com.hubspot.jinjava.el.ext;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.hubspot.jinjava.Jinjava;
+import com.hubspot.jinjava.interpret.FatalTemplateErrorsException;
+
+public class TruncDivTest {
+
+    @Before
+    public void setUp() {
+        jinja = new Jinjava();
+    }
+
+    /**
+     * Test the truncated division operator "//" with integer values
+     */
+    @Test
+    public void testTruncDivInteger() {
+        Map<String, Object> context = Maps.newHashMap();
+        context.put("dividend", 5);
+        context.put("divisor",  2);
+
+        String template = "{% set x = dividend // divisor %}{{x}}";
+        String rendered = jinja.render(template, context);
+        assertEquals("2", rendered);
+    }
+
+    /**
+     * Test the truncated division operator "//" with fractional values
+     */
+    @Test
+    public void testTruncDivFractional() {
+
+        Map<String, Object> context = Maps.newHashMap();
+        context.put("dividend", 5.0);
+        context.put("divisor",  2);
+
+        String template = "{% set x = dividend // divisor %}{{x}}";
+        String rendered = jinja.render(template, context);
+        assertEquals("2.0", rendered);
+    }
+
+    /**
+     * Test the truncated division operator "//" with strings
+     */
+    @Test
+    public void testTruncDivStringFails() {
+
+        Map<String, Object> context = Maps.newHashMap();
+        context.put("dividend", "5");
+        context.put("divisor",  "2");
+
+        String template = "{% set x = dividend // divisor %}{{x}}";
+        try {
+            jinja.render(template, context);
+        } catch (FatalTemplateErrorsException e) {
+            String msg = e.getMessage();
+            assertTrue(msg.contains("Unsupported operand type(s)"));
+        }
+    }
+
+
+    private Jinjava jinja;
+}

src/test/java/com/hubspot/jinjava/lib/tag/ForTagTest.java

@@ -1,6 +1,7 @@
 package com.hubspot.jinjava.lib.tag;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -31,10 +32,12 @@ public class ForTagTest {
 
   Context context;
   JinjavaInterpreter interpreter;
+  Jinjava jinjava;
 
   @Before
   public void setup() {
-    interpreter = new Jinjava().newInterpreter();
+    jinjava = new Jinjava();
+    interpreter = jinjava.newInterpreter();
     context = interpreter.getContext();
 
     tag = new ForTag();
@@ -113,6 +116,59 @@ public void forLoopSupportsAllLoopVarsInHublDocs() throws Exception {
     assertThat(dom.select(".item-0 .subnum").text()).isEqualTo("6 0");
   }
 
+  @Test
+  public void testForLoopConstants() {
+
+      Map<String, Object> context = Maps.newHashMap();
+      String template = ""
+              + "{% for i in range(1 * 1, 2 * 2) %}{{i}}{% endfor %}";
+
+      String rendered = jinjava.render(template, context);
+      assertEquals("123", rendered);
+  }
+
+  @Test
+  public void testForLoopVariablesWithoutSpaces() {
+
+      Map<String, Object> context = Maps.newHashMap();
+      context.put("a", 2);
+      context.put("b", 3);
+
+      String template = ""
+          + "{% for index in range(a*b,a*b+b) %}"
+          + "{{index}} "
+          + "{% endfor %}";
+
+      String rendered = jinjava.render(template, context);
+      assertEquals("6 7 8 ", rendered);
+  }
+
+  @Test
+  public void testFoorLoopVariablesWithSpaces() {
+
+      Map<String, Object> context = Maps.newHashMap();
+      context.put("a", 2);
+      context.put("b", 3);
+
+      String template = ""
+          + "{% for index in range(a * b, a * b + b) %}"
+          + "{{index}} "
+          + "{% endfor %}";
+
+      String rendered = jinjava.render(template, context);
+      assertEquals("6 7 8 ", rendered);
+  }
+
+  @Test
+  public void testForLoopRangeWithStringsWithSpaces() {
+      Map<String, Object> context = Maps.newHashMap();
+      String template = ""
+           + "{% for i in ['a ','b'] %}{{i}}{% endfor %}";
+      String rendered = jinjava.render(template, context);
+      System.out.println(rendered);
+      assertEquals("a b", rendered);
+  }
+
   private Node fixture(String name) {
     try {
       return new TreeParser(interpreter, Resources.toString(