feat: Add support for Starlark float literals (#5678)

* feat: Add support for Starlark float literals * Add additional test
bazelbuild · Nov 16, 2023 · 1fc8ec6 · 1fc8ec6
1 parent b467f58
commit 1fc8ec6
Show file tree

Hide file tree

Showing 11 changed files with 137 additions and 37 deletions.
diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/highlighting/BuildSyntaxHighlighter.java b/base/src/com/google/idea/blaze/base/lang/buildfile/highlighting/BuildSyntaxHighlighter.java
@@ -75,6 +75,7 @@ private static TextAttributesKey key(String name, TextAttributesKey fallbackKey)
   static {
     addAttribute(TokenKind.COMMENT, BUILD_LINE_COMMENT);
     addAttribute(TokenKind.INT, BUILD_NUMBER);
+    addAttribute(TokenKind.FLOAT, BUILD_NUMBER);
     addAttribute(TokenKind.STRING, BUILD_STRING);
 
     addAttribute(TokenKind.LBRACE, BUILD_BRACES);

diff --git a/...rc/com/google/idea/blaze/base/lang/buildfile/language/semantics/BuiltInNamesProvider.java b/...rc/com/google/idea/blaze/base/lang/buildfile/language/semantics/BuiltInNamesProvider.java
@@ -41,6 +41,7 @@ public class BuiltInNamesProvider {
           "enumerate",
           "fail",
           "False",
+          "float",
           "getattr",
           "hasattr",
           "hash",

diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/lexer/BuildLexerBase.java b/base/src/com/google/idea/blaze/base/lang/buildfile/lexer/BuildLexerBase.java
@@ -19,6 +19,7 @@
 import com.google.common.collect.Lists;
 import java.util.List;
 import java.util.Stack;
+import java.util.function.Predicate;
 import javax.annotation.Nullable;
 
 /**
@@ -559,8 +560,32 @@ private void addIdentifierOrKeyword() {
     addToken(kind, oldPos, pos, (kind == TokenKind.IDENTIFIER) ? id : null);
   }
 
+  private String scanDecimal() {
+    int oldPos = pos;
+    while (pos < buffer.length) {
+      char c = buffer[pos];
+      switch (c) {
+        case '0':
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+        case '5':
+        case '6':
+        case '7':
+        case '8':
+        case '9':
+          pos++;
+          break;
+        default:
+          return bufferSlice(oldPos, pos);
+      }
+    }
+    return bufferSlice(oldPos, pos);
+  }
+
   private String scanInteger() {
-    int oldPos = pos - 1;
+    int oldPos = pos;
     while (pos < buffer.length) {
       char c = buffer[pos];
       switch (c) {
@@ -600,17 +625,21 @@ private String scanInteger() {
   }
 
   /**
-   * Scans an addInteger literal.
+   * Scans an integer or float literal.
    *
-   * <p>ON ENTRY: 'pos' is 1 + the index of the first char in the literal. ON EXIT: 'pos' is 1 + the
+   * <p>ON ENTRY: 'pos' is the index of the first char in the literal. ON EXIT: 'pos' is 1 + the
    * index of the last char in the literal.
    */
-  private void addInteger() {
-    int oldPos = pos - 1;
+  private void addNumber() {
+    // https://github.com/bazelbuild/starlark/blob/master/spec.md#lexical-elements
+
+    int oldPos = pos;
     String literal = scanInteger();
 
-    final String substring;
-    final int radix;
+    String substring;
+    int radix;
+    boolean isFloat = false;
+    boolean hasErrors = false;
     if (literal.startsWith("0x") || literal.startsWith("0X")) {
       radix = 16;
       substring = literal.substring(2);
@@ -626,14 +655,50 @@ private void addInteger() {
       substring = literal;
     }
 
-    int value = 0;
-    try {
-      value = Integer.parseInt(substring, radix);
-    } catch (NumberFormatException e) {
-      error("invalid base-" + radix + " integer constant: " + literal);
+    if (radix == 10 && lookaheadIs(0, '.')) {
+      pos++;
+      isFloat = true;
+      scanDecimal();
+      literal = bufferSlice(oldPos, pos);
+      substring = literal;
+    }
+    if (radix == 10 && (lookaheadIs(0, 'e') || lookaheadIs(0, 'E'))) {
+      pos++;
+      isFloat = true;
+      if (lookaheadIs(0, '+') || lookaheadIs(0, '-')) {
+        pos++;
+      }
+      String exp = scanDecimal();
+      if (exp.isEmpty()) {
+        hasErrors = true;
+      }
+      literal = bufferSlice(oldPos, pos);
+      substring = literal;
     }
 
-    addToken(TokenKind.INT, oldPos, pos, value);
+    if (isFloat) {
+      double value = 0;
+      try {
+        value = Double.parseDouble(substring);
+      } catch (NumberFormatException e) {
+        hasErrors = true;
+      }
+      if (hasErrors) {
+        error("invalid float constant: " + literal, oldPos, pos);
+      }
+      addToken(TokenKind.FLOAT, oldPos, pos, value);
+    } else {
+      int value = 0;
+      try {
+        value = Integer.parseInt(substring, radix);
+      } catch (NumberFormatException e) {
+        hasErrors = true;
+      }
+      if (hasErrors) {
+        error("invalid base-" + radix + " integer constant: " + literal);
+      }
+      addToken(TokenKind.INT, oldPos, pos, value);
+    }
   }
 
   /**
@@ -665,6 +730,10 @@ private boolean lookaheadIs(int p, char c) {
     return pos + p < buffer.length && buffer[pos + p] == c;
   }
 
+  private boolean lookaheadMatches(int p, Predicate<Character> pred) {
+    return pos + p < buffer.length && pred.test(buffer[pos + p]);
+  }
+
   /** Performs tokenization of the character buffer of file contents provided to the constructor. */
   private void tokenize() {
     while (pos < buffer.length) {
@@ -741,9 +810,6 @@ private void tokenize() {
         case ';':
           addToken(TokenKind.SEMI, pos - 1, pos);
           break;
-        case '.':
-          addToken(TokenKind.DOT, pos - 1, pos);
-          break;
         case '*':
           addToken(TokenKind.STAR, pos - 1, pos);
           break;
@@ -791,8 +857,12 @@ private void tokenize() {
             break;
           }
 
-          if (Character.isDigit(c)) {
-            addInteger();
+          // Distinguish dot vs. start of a float.
+          if (c == '.' && !lookaheadMatches(0, Character::isDigit)) {
+            addToken(TokenKind.DOT, pos - 1, pos);
+          } else if (Character.isDigit(c) || c == '.') {
+            pos--;
+            addNumber();
           } else if (Character.isJavaIdentifierStart(c) && c != '$') {
             addIdentifierOrKeyword();
           } else {

diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/lexer/TokenKind.java b/base/src/com/google/idea/blaze/base/lang/buildfile/lexer/TokenKind.java
@@ -38,6 +38,7 @@ public enum TokenKind {
   EQUALS_EQUALS("=="),
   EXCEPT("except"),
   FINALLY("finally"),
+  FLOAT("float"),
   FOR("for"),
   FROM("from"),
   GLOBAL("global"),

diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/parser/ExpressionParsing.java b/base/src/com/google/idea/blaze/base/lang/buildfile/parser/ExpressionParsing.java
@@ -249,6 +249,9 @@ private void parsePrimary() {
       case INT:
         buildTokenElement(BuildElementTypes.INTEGER_LITERAL);
         return;
+      case FLOAT:
+        buildTokenElement(BuildElementTypes.FLOAT_LITERAL);
+        return;
       case STRING:
         parseStringLiteral(true);
         return;

diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/psi/BuildElementTypes.java b/base/src/com/google/idea/blaze/base/lang/buildfile/psi/BuildElementTypes.java
@@ -71,6 +71,7 @@ public interface BuildElementTypes {
   BuildElementType DOT_EXPRESSION = new BuildElementType("dot_expr", DotExpression.class);
   BuildElementType STRING_LITERAL = new BuildElementType("string", StringLiteral.class);
   BuildElementType INTEGER_LITERAL = new BuildElementType("int", IntegerLiteral.class);
+  BuildElementType FLOAT_LITERAL = new BuildElementType("float", FloatLiteral.class);
   BuildElementType LIST_LITERAL = new BuildElementType("list", ListLiteral.class);
   BuildElementType GLOB_EXPRESSION = new BuildElementType("glob", GlobExpression.class);
   BuildElementType REFERENCE_EXPRESSION =
@@ -92,6 +93,7 @@ public interface BuildElementTypes {
           DOT_EXPRESSION,
           STRING_LITERAL,
           INTEGER_LITERAL,
+          FLOAT_LITERAL,
           LIST_LITERAL,
           PARENTHESIZED_EXPRESSION,
           TUPLE_EXPRESSION,

diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/psi/BuildElementVisitor.java b/base/src/com/google/idea/blaze/base/lang/buildfile/psi/BuildElementVisitor.java
@@ -104,6 +104,10 @@ public void visitIntegerLiteral(IntegerLiteral node) {
     visitElement(node);
   }
 
+  public void visitFloatLiteral(FloatLiteral node) {
+    visitElement(node);
+  }
+
   public void visitListLiteral(ListLiteral node) {
     visitElement(node);
   }

diff --git a/base/src/com/google/idea/blaze/base/lang/buildfile/psi/FloatLiteral.java b/base/src/com/google/idea/blaze/base/lang/buildfile/psi/FloatLiteral.java
@@ -0,0 +1,15 @@
+package com.google.idea.blaze.base.lang.buildfile.psi;
+
+import com.intellij.lang.ASTNode;
+
+public class FloatLiteral extends BuildElementImpl implements LiteralExpression {
+
+  public FloatLiteral(ASTNode astNode) {
+    super(astNode);
+  }
+
+  @Override
+  protected void acceptVisitor(BuildElementVisitor visitor) {
+    visitor.visitFloatLiteral(this);
+  }
+}
diff --git a/base/tests/unittests/com/google/idea/blaze/base/lang/buildfile/lexer/BlazeLexerTest.java b/base/tests/unittests/com/google/idea/blaze/base/lang/buildfile/lexer/BlazeLexerTest.java
@@ -63,28 +63,29 @@ public void testBasics4() throws Exception {
     assertEquals("", names(tokens("")));
     assertEquals("", names(tokens("# foo")));
     assertEquals("INT INT INT INT", names(tokens("1 2 3 4")));
-    assertEquals("INT DOT INT", names(tokens("1.234")));
+    assertEquals("FLOAT", names(tokens("1.234")));
     assertEquals(
         "IDENTIFIER LPAREN IDENTIFIER COMMA IDENTIFIER RPAREN", names(tokens("foo(bar, wiz)")));
   }
 
   @Test
   public void testIntegersAndDot() throws Exception {
-    assertEquals("INT(1) DOT INT(2345)", values(tokens("1.2345")));
+    assertEquals("FLOAT(1.2345)", values(tokens("1.2345")));
 
-    assertEquals("INT(1) DOT INT(2) DOT INT(345)", values(tokens("1.2.345")));
+    assertEquals("FLOAT(1.2) FLOAT(0.345)", values(tokens("1.2.345")));
 
-    assertEquals("INT(1) DOT INT(0)", values(tokens("1.23E10")));
-    assertEquals("invalid base-10 integer constant: 23E10", lastError);
+    assertEquals("FLOAT(1.23E10)", values(tokens("1.23E10")));
 
-    assertEquals("INT(1) DOT INT(0) MINUS INT(10)", values(tokens("1.23E-10")));
-    assertEquals("invalid base-10 integer constant: 23E", lastError);
+    assertEquals("FLOAT(0.0)", values(tokens("1.23E")));
+    assertEquals("invalid float constant: 1.23E", lastError);
 
+    assertEquals("FLOAT(1.23E-10)", values(tokens("1.23E-10")));
     assertEquals("DOT INT(123)", values(tokens(". 123")));
-    assertEquals("DOT INT(123)", values(tokens(".123")));
+    assertEquals("FLOAT(0.123)", values(tokens(".123")));
     assertEquals("DOT IDENTIFIER(abc)", values(tokens(".abc")));
 
-    assertEquals("IDENTIFIER(foo) DOT INT(123)", values(tokens("foo.123")));
+    assertEquals("IDENTIFIER(foo) FLOAT(0.123)", values(tokens("foo.123")));
+    assertEquals("IDENTIFIER(foo456) FLOAT(0.123)", values(tokens("foo456.123")));
     assertEquals(
         "IDENTIFIER(foo) DOT IDENTIFIER(bcd)", values(tokens("foo.bcd"))); // 'b' are hex chars
     assertEquals("IDENTIFIER(foo) DOT IDENTIFIER(xyz)", values(tokens("foo.xyz")));

diff --git a/...ests/unittests/com/google/idea/blaze/base/lang/buildfile/lexer/HighlightingLexerTest.java b/...ests/unittests/com/google/idea/blaze/base/lang/buildfile/lexer/HighlightingLexerTest.java
@@ -63,29 +63,31 @@ public void testBasics4() throws Exception {
     assertEquals("", names(tokens("")));
     assertEquals("COMMENT", names(tokens("# foo")));
     assertEquals("INT WHITESPACE INT WHITESPACE INT WHITESPACE INT", names(tokens("1 2 3 4")));
-    assertEquals("INT DOT INT", names(tokens("1.234")));
+    assertEquals("FLOAT", names(tokens("1.234")));
+    assertEquals("DOT IDENTIFIER", names(tokens(".a")));
     assertEquals(
         "IDENTIFIER LPAREN IDENTIFIER COMMA WHITESPACE IDENTIFIER RPAREN",
         names(tokens("foo(bar, wiz)")));
   }
 
   @Test
   public void testIntegersAndDot() throws Exception {
-    assertEquals("INT(1) DOT INT(2345)", values(tokens("1.2345")));
+    assertEquals("FLOAT(1.2345)", values(tokens("1.2345")));
 
-    assertEquals("INT(1) DOT INT(2) DOT INT(345)", values(tokens("1.2.345")));
+    assertEquals("FLOAT(1.2) FLOAT(0.345)", values(tokens("1.2.345")));
 
-    assertEquals("INT(1) DOT INT(0)", values(tokens("1.23E10")));
-    assertEquals("invalid base-10 integer constant: 23E10", lastError);
+    assertEquals("FLOAT(1.23E10)", values(tokens("1.23E10")));
 
-    assertEquals("INT(1) DOT INT(0) MINUS INT(10)", values(tokens("1.23E-10")));
-    assertEquals("invalid base-10 integer constant: 23E", lastError);
+    assertEquals("FLOAT(0.0)", values(tokens("1.23E")));
+    assertEquals("invalid float constant: 1.23E", lastError);
 
+    assertEquals("FLOAT(1.23E-10)", values(tokens("1.23E-10")));
     assertEquals("DOT WHITESPACE INT(123)", values(tokens(". 123")));
-    assertEquals("DOT INT(123)", values(tokens(".123")));
+    assertEquals("FLOAT(0.123)", values(tokens(".123")));
     assertEquals("DOT IDENTIFIER(abc)", values(tokens(".abc")));
 
-    assertEquals("IDENTIFIER(foo) DOT INT(123)", values(tokens("foo.123")));
+    assertEquals("IDENTIFIER(foo) FLOAT(0.123)", values(tokens("foo.123")));
+    assertEquals("IDENTIFIER(foo456) FLOAT(0.123)", values(tokens("foo456.123")));
     assertEquals(
         "IDENTIFIER(foo) DOT IDENTIFIER(bcd)", values(tokens("foo.bcd"))); // 'b' are hex chars
     assertEquals("IDENTIFIER(foo) DOT IDENTIFIER(xyz)", values(tokens("foo.xyz")));

diff --git a/skylark/src/com/google/idea/blaze/skylark/debugger/impl/SkylarkDebugValue.java b/skylark/src/com/google/idea/blaze/skylark/debugger/impl/SkylarkDebugValue.java
@@ -61,7 +61,7 @@ static Type parseType(StarlarkDebuggingProtos.Value value) {
   // TODO(brendandouglas): move this logic onto the server side?
   private static final ImmutableSet<String> ARRAY_TYPES = ImmutableSet.of("dict", "list", "depset");
   private static final ImmutableSet<String> PRIMITIVE_TYPES =
-      ImmutableSet.of("bool", "string", "int");
+      ImmutableSet.of("bool", "string", "int", "float");
   private static final ImmutableSet<String> FUNCTION_TYPES =
       ImmutableSet.of("function", "Provider");