Merge pull request #354 from alephium/b_string_parser

`BStringParser`

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringParser.scala

@@ -0,0 +1,31 @@
+package org.alephium.ralph.lsp.access.compiler.parser.soft
+
+import fastparse._
+import fastparse.NoWhitespace.noWhitespaceImplicit
+import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
+
+private object BStringParser {
+
+  def parseOrFail[Unknown: P]: P[SoftAST.BString] =
+    P {
+      Index ~
+        TokenParser.parseOrFail(Token.B) ~
+        SpaceParser.parseOrFail.? ~
+        TokenParser.parseOrFail(Token.Tick) ~
+        TextParser.parseOrFail(Token.Tick).? ~
+        TokenParser.parse(Token.Tick) ~
+        Index
+    } map {
+      case (from, b, postBSpace, startTick, text, endTick, to) =>
+        SoftAST.BString(
+          index = range(from, to),
+          b = b,
+          postBSpace = postBSpace,
+          startTick = startTick,
+          text = text,
+          endTick = endTick
+        )
+    }
+
+}

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/Demo.scala

@@ -44,6 +44,11 @@ object Demo extends App {
         |
         |     // mutable binding
         |     let (a, mut b, _) = (1, 2, 3)
+        |
+        |     let string =
+        |       b`some text 🎸
+        |         some text in the middle
+        |         some more text 🤙`
         |  }
         |
         |  🚀

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala

@@ -101,6 +101,7 @@ private object ExpressionParser {
         TupleParser.parseOrFail |
         NumberParser.parseOrFail |
         BooleanParser.parseOrFail |
+        BStringParser.parseOrFail |
         IdentifierParser.parseOrFail
     }
 

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/SoftAST.scala

@@ -429,6 +429,15 @@ object SoftAST {
       unit: Option[TokenDocumented[Token.AlphLowercase.type]])
     extends ExpressionAST
 
+  case class BString(
+      index: SourceIndex,
+      b: TokenDocumented[Token.B.type],
+      postBSpace: Option[Space],
+      startTick: TokenDocumented[Token.Tick.type],
+      text: Option[CodeString],
+      endTick: TokenDocExpectedAST[Token.Tick.type])
+    extends ExpressionAST
+
   sealed trait SpaceAST extends SoftAST
 
   case class Space(

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala

@@ -96,6 +96,7 @@ object Token {
   case object At                                                extends Punctuator("@") with Reserved
   case object Tick                                              extends Punctuator("`") with Reserved
   case object Quote                                             extends Punctuator("\"") with Reserved
+  case object B                                                 extends Punctuator("b") with Token
 
   sealed abstract class Data(override val lexeme: String) extends Token
   case object Const                                       extends Data("const") with Reserved

compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala

@@ -0,0 +1,167 @@
+package org.alephium.ralph.lsp.access.compiler.parser.soft
+
+import org.alephium.ralph.lsp.access.compiler.parser.soft.TestParser._
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.TestSoftAST._
+import org.alephium.ralph.lsp.access.util.TestCodeUtil._
+import org.alephium.ralph.lsp.access.util.TestFastParse.assertIsFastParseError
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+import org.scalatest.OptionValues._
+
+class BStringSpec extends AnyWordSpec with Matchers {
+
+  "b alone" should {
+    "not parse as String literal" in {
+      // this is because "b" is not a reserved token.
+      // "b" followed by tick is required for string literal.
+      assertIsFastParseError {
+        parseBString("b")
+      }
+    }
+  }
+
+  "b followed by a tick" should {
+    "parse as string literal" in {
+      // this is successfully parsed and the ending tick is reported as missing.
+      val string =
+        parseBString("b`") // closing tick is missing
+
+      string shouldBe
+        SoftAST.BString(
+          index = indexOf(">>b`<<"),
+          b = B(indexOf(">>b<<`")),
+          postBSpace = None,
+          startTick = Tick(indexOf("b>>`<<")),
+          text = None,
+          endTick = SoftAST.TokenExpected(indexOf("b`>><<"), Token.Tick)
+        )
+    }
+  }
+
+  "empty string" in {
+    val string =
+      parseBString("b``")
+
+    string shouldBe
+      SoftAST.BString(
+        index = indexOf(">>b``<<"),
+        b = B(indexOf(">>b<<``")),
+        postBSpace = None,
+        startTick = Tick(indexOf("b>>`<<`")),
+        text = None,
+        endTick = Tick(indexOf("b`>>`<<"))
+      )
+  }
+
+  "spaces only" in {
+    val string =
+      parseBString("b ` `")
+
+    string shouldBe
+      SoftAST.BString(
+        index = indexOf(">>b ` `<<"),
+        b = B(indexOf(">>b<< ` `")),
+        postBSpace = Some(SpaceOne(indexOf("b>> <<` `"))),
+        startTick = Tick(indexOf("b >>`<< `")),
+        text = Some(SoftAST.CodeString(indexOf("b `>> <<`"), " ")),
+        endTick = Tick(indexOf("b ` >>`<<"))
+      )
+  }
+
+  "comment exist before b" in {
+    val string =
+      parseBString {
+        """// this is a byte string
+          |b ` `""".stripMargin
+      }
+
+    // Simply check that the comment exists before b.
+    // No need to assert the comment AST, CommentsSpec should assert this.
+    val comments =
+      string.b.documentation.value
+
+    comments.index shouldBe
+      indexOf {
+        """>>// this is a byte string
+          |<<b ` `""".stripMargin
+      }
+
+    comments.toCode() shouldBe
+      """// this is a byte string
+        |""".stripMargin
+
+  }
+
+  "comment exist before tick" in {
+    val string =
+      parseBString {
+        """b
+          |// this is a byte string
+          |` `""".stripMargin
+      }
+
+    // Check that the comment exists before b.
+    // No need to assert the comment AST, CommentsSpec should assert this.
+    val comments =
+      string.startTick.documentation.value
+
+    comments.index shouldBe
+      indexOf {
+        """b
+          |>>// this is a byte string
+          |<<` `""".stripMargin
+      }
+
+    comments.toCode() shouldBe
+      """// this is a byte string
+        |""".stripMargin
+
+  }
+
+  "string characters exist" when {
+    "closing tick is missing" should {
+      "parse the entire string" in {
+        val string =
+          parseBString {
+            """b` this is
+              |
+              |a string value
+              |""".stripMargin
+          }
+
+        string.text.value shouldBe
+          SoftAST.CodeString(
+            index = indexOf {
+              """b`>> this is
+                |
+                |a string value
+                |<<""".stripMargin
+            },
+            text = """ this is
+                |
+                |a string value
+                |""".stripMargin
+          )
+
+        string.text.value.text shouldBe
+          """ this is
+            |
+            |a string value
+            |""".stripMargin
+
+        string.endTick shouldBe
+          SoftAST.TokenExpected(
+            index = indexOf {
+              """b` this is
+                |
+                |a string value
+                |>><<""".stripMargin
+            },
+            token = Token.Tick
+          )
+      }
+    }
+  }
+
+}

compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala

@@ -78,6 +78,9 @@ object TestParser {
   def parseNumber(code: String): SoftAST.Number =
     runSoftParser(NumberParser.parseOrFail(_))(code)
 
+  def parseBString(code: String): SoftAST.BString =
+    runSoftParser(BStringParser.parseOrFail(_))(code)
+
   def findAnnotation(identifier: String)(code: String): Option[SoftAST.Annotation] =
     findAnnotation(
       identifier = identifier,

compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/TestSoftAST.scala

@@ -126,6 +126,18 @@ object TestSoftAST {
       token = Token.Plus
     )
 
+  def B(index: SourceIndex): SoftAST.TokenDocumented[Token.B.type] =
+    TokenDocumented(
+      index = index,
+      token = Token.B
+    )
+
+  def Tick(index: SourceIndex): SoftAST.TokenDocumented[Token.Tick.type] =
+    TokenDocumented(
+      index = index,
+      token = Token.Tick
+    )
+
   def Identifier(
       index: SourceIndex,
       text: String): SoftAST.Identifier =