Update to the parser following grammar update

modules/formatter/src/ast/node.scala

@@ -89,9 +89,6 @@ object QuotedChar {
 
   case class EscapedCharCase(char: EscapedChar) extends QuotedChar
 
-  case class PreservedDoubleCase(preservedDouble: PreservedDouble)
-      extends QuotedChar
-
   case object NewLineCase extends QuotedChar
 }
 
@@ -107,6 +104,6 @@ object EscapedChar {
       extends EscapedChar
 }
 
-case class PreservedDouble(char: Char)
+case class TextBlock(content: List[TextBlockContent])
 
-case class TextBlock(quotedChars: List[QuotedChar])
+case class TextBlockContent(dquotes: List[Char], char: QuotedChar)

modules/formatter/src/ast/shapes.scala

@@ -29,7 +29,7 @@ object shapes {
   // UseSection and ShapeStatements are both Optional , however encoded via Lists . see https://github.com/awslabs/smithy/issues/1249
   case class ShapeSection(
       all: Option[
-        (NamespaceStatement, UseSection, ShapeStatements)
+        (NamespaceStatement, UseSection, Option[ShapeStatements])
       ]
   )
 
@@ -45,7 +45,10 @@ object shapes {
       break: Break
   )
 
-  case class ShapeStatements(statements: List[ShapeStatementsCase])
+  case class ShapeStatements(
+      first: ShapeStatementsCase,
+      others: List[(Break, ShapeStatementsCase)]
+  )
 
   sealed trait ShapeStatementsCase
 
@@ -59,8 +62,7 @@ object shapes {
 
   case class ShapeStatement(
       traitStatements: TraitStatements,
-      shapeBody: ShapeBody,
-      br: Break
+      shapeBody: ShapeBody
   )
 
   /*  case class ShapeMemberKvp(
@@ -117,8 +119,6 @@ object shapes {
 
       case class ExplicitListMember(shapeId: ShapeId) extends ListMemberType
 
-      // Diverging from the grammer: https://smithy.io/2.0/spec/idl.html#grammar-token-smithy-ListMember
-      // The grammar says the members is mandatory, but it isnt
       case class ListMembers(
           ws0: Whitespace,
           members: Option[ListMember],
@@ -165,8 +165,6 @@ object shapes {
         case class ExplicitMapValue(shapeId: ShapeId) extends MapValueType
       }
 
-      // Diverging from the grammar: https://smithy.io/2.0/spec/idl.html#grammar-token-smithy-MapMembers
-      // The grammar says key member and value member are mandatory but they are not in practice
       case class MapMembers(
           ws0: Whitespace,
           mapKey: Option[MapKey],

modules/formatter/src/ast/smithyTrait.scala

@@ -64,14 +64,12 @@ case class ApplyStatementSingular(
     ws0: Whitespace,
     shapeId: ShapeId,
     ws1: Whitespace,
-    strait: SmithyTrait,
-    break: Break
+    strait: SmithyTrait
 )
 case class ApplyStatementBlock(
     shapeId: ShapeId,
     ws0: Whitespace,
-    traitStatements: TraitStatements,
-    break: Break
+    traitStatements: TraitStatements
 )
 case class ApplyStatement(
     either: Either[ApplyStatementSingular, ApplyStatementBlock]

modules/formatter/src/parsers/IdlParser.scala

@@ -26,9 +26,9 @@ import ShapeParser.shape_section
 object IdlParser {
 
   val idlParser: Parser0[Idl] =
-    (ws ~ control_section ~ metadata_section ~ shape_section).map {
-      case (((whitespace, control), metadata), shape) =>
+    (ws ~ control_section ~ metadata_section ~ shape_section)
+      .map { case (((whitespace, control), metadata), shape) =>
         Idl(whitespace, control, metadata, shape)
-    }
+      }
 
 }

modules/formatter/src/parsers/NodeParser.scala

@@ -30,45 +30,45 @@ import smithytranslate.formatter.ast.NodeValue._
 import smithytranslate.formatter.ast.QuotedChar.{
   EscapedCharCase,
   NewLineCase,
-  PreservedDoubleCase,
   SimpleCharCase
 }
 import smithytranslate.formatter.parsers.WhitespaceParser.{nl, sp0, ws}
 import smithytranslate.formatter.ast.{
   EscapedChar,
   NodeValue,
-  PreservedDouble,
   QuotedChar,
   QuotedText,
-  TextBlock
+  TextBlock,
+  TextBlockContent
 }
 import smithytranslate.formatter.parsers.ShapeIdParser.{identifier, shape_id}
 
 object NodeParser {
 
   val opParser: Parser[Char] = Parser.charIn(op)
   val qChar: Parser[Char] = Parser.charIn(allQuotable)
-  val preserved_double: Parser[PreservedDouble] = escape *> qChar
-    .map(PreservedDouble)
   val unicode_escape: Parser[UnicodeEscape] =
     (Parser.char('u') *> hexdig ~ hexdig ~ hexdig).map { case ((a, b), c) =>
       UnicodeEscape(a, b, c)
     }
 
   val escaped_char: Parser[EscapedChar] =
     escape *> (Parser.charIn(escapeChars).map(CharCase) | unicode_escape)
-  val QuotedChar: Parser[QuotedChar] =
+  val quoted_char: Parser[QuotedChar] =
     qChar.backtrack.map(SimpleCharCase) |
       escaped_char.backtrack.map(EscapedCharCase) |
-      preserved_double.map(PreservedDoubleCase) |
       nl.as(NewLineCase)
 
-  val ThreeDquotes = dquote ~ dquote ~ dquote
+  val three_dquotes: Parser[Unit] = dquote *> dquote *> dquote
+  val text_block_content: Parser[TextBlockContent] =
+    (Parser.charIn('"').rep0(0, 2).soft.with1 ~ quoted_char).map {
+      case (quotes, char) => TextBlockContent(quotes, char)
+    }
   val text_block: Parser[TextBlock] =
-    ((ThreeDquotes ~ sp0 *> nl) *> QuotedChar.rep0 <* ThreeDquotes)
+    ((three_dquotes ~ sp0 *> nl) *> text_block_content.rep0 <* three_dquotes)
       .map(TextBlock)
   val quoted_text: Parser[QuotedText] =
-    (dquote *> QuotedChar.rep0 <* dquote).map(QuotedText)
+    (dquote *> quoted_char.rep0 <* dquote).map(QuotedText)
   val node_string_value: Parser[NodeStringValue] = shape_id.backtrack.map(
     ShapeIdCase
   ) | text_block.backtrack.map(TextBlockCase) | quoted_text.backtrack
@@ -177,32 +177,32 @@ QuotedText =
     DQUOTE *QuotedChar DQUOTE
 
 QuotedChar =
-    %x20-21     ; space - "!"
+    %x09        ; tab
+  / %x20-21     ; space - "!"
   / %x23-5B     ; "#" - "["
   / %x5D-10FFFF ; "]"+
   / EscapedChar
-  / PreservedDouble
   / NL
 
-EscapedChar =
-    Escape (Escape / "'" / DQUOTE / %s"b"
-            / %s"f" / %s"n" / %s"r" / %s"t"
-            / "/" / UnicodeEscape)
+  EscapedChar =
+      Escape (Escape / DQUOTE / %s"b" / %s"f"
+    / %s"n" / %s"r" / %s"t" / "/"
+    / UnicodeEscape
 
 UnicodeEscape =
     %s"u" Hex Hex Hex Hex
 
 Hex =
     DIGIT / %x41-46 / %x61-66
 
-PreservedDouble =
-    Escape (%x20-21 / %x23-5B / %x5D-10FFFF)
-
 Escape =
     %x5C ; backslash
 
 TextBlock =
-    ThreeDquotes *SP NL *QuotedChar ThreeDquotes
+    ThreeDquotes *SP NL *TextBlockContent ThreeDquotes
+
+TextBlockContent =
+    QuotedChar / (1*2DQUOTE 1*QuotedChar)
 
 ThreeDquotes =
     DQUOTE DQUOTE DQUOTE

modules/formatter/src/parsers/ShapeParser.scala

@@ -17,7 +17,7 @@ package formatter
 package parsers
 
 import cats.parse.{Parser, Parser0}
-import smithytranslate.formatter.ast.{ShapeId, Break, Whitespace, shapes}
+import smithytranslate.formatter.ast.{ShapeId, Whitespace, shapes}
 import smithytranslate.formatter.ast.shapes._
 import smithytranslate.formatter.ast.shapes.ShapeBody._
 import smithytranslate.formatter.ast.shapes.ShapeBody.ListStatement.{
@@ -339,21 +339,21 @@ object ShapeParser {
   val shape_body: Parser[shapes.ShapeBody] =
     simple_shape_statement | enum_shape_statement | list_statement | set_statement | map_statement | structure_statement | union_statement | service_statement | operation_statement | resource_statement
   val shape_statement: Parser[ShapeStatement] = {
-    val traitAndBody = trait_statements.with1 ~ shape_body
-    val interspersedBr =
-      (traitAndBody.soft ~ br) | traitAndBody.map(_ -> Break(None, Nil))
-    interspersedBr.map { case ((a, b), c) =>
-      ShapeStatement(a, b, c)
+    (trait_statements.with1 ~ shape_body).map { case (a, b) =>
+      ShapeStatement(a, b)
     }
   }
-  val shape_statements: Parser0[ShapeStatements] = (shape_statement
-    .map(
-      ShapeStatementCase
-    ) | apply_statement.map(
-    ApplyStatementCase
-  )).rep0.map {
-    ShapeStatements
-  }
+  val shape_statement_or_apply: Parser[ShapeStatementsCase] =
+    shape_statement.map(ShapeStatementCase) |
+      apply_statement.map(ApplyStatementCase)
+
+  // The optional trailing BR is not in the spec but it exists in a lot of
+  // files.
+  val shape_statements: Parser0[ShapeStatements] =
+    (shape_statement_or_apply ~ (br ~ shape_statement_or_apply).backtrack.rep0 <* (ws.? *> br.?))
+      .map { case (firstStatement, others) =>
+        ShapeStatements(firstStatement, others)
+      }
 
   val namespace_statement: Parser[NamespaceStatement] =
     Parser.string("namespace") *> (sp *> namespace ~ br).map { case (ns, br) =>
@@ -365,7 +365,7 @@ object ShapeParser {
     }
   val use_section: Parser0[UseSection] = use_statement.rep0.map(UseSection)
   val shape_section: Parser0[ShapeSection] =
-    (namespace_statement ~ use_section ~ shape_statements).?.map { op =>
+    (namespace_statement ~ use_section ~ shape_statements.?).?.map { op =>
       ShapeSection(op.map { case ((ns, use), ss) =>
         (ns, use, ss)
       })
@@ -374,7 +374,7 @@ object ShapeParser {
 
 /*
 ShapeSection =
-    [NamespaceStatement UseSection ShapeStatements]
+    [NamespaceStatement UseSection [ShapeStatements]]
 
 NamespaceStatement =
     %s"namespace" SP Namespace BR
@@ -386,10 +386,13 @@ UseStatement =
     %s"use" SP AbsoluteRootShapeId BR
 
 ShapeStatements =
- *(ShapeStatement / ApplyStatement)
+  ShapeOrApplyStatement *(BR ShapeOrApplyStatement)
+
+ShapeOrApplyStatement =
+  ShapeStatement / ApplyStatement
 
 ShapeStatement =
-    TraitStatements ShapeBody BR
+  TraitStatements ShapeBody
 
 ShapeBody =
     SimpleShapeStatement
@@ -445,7 +448,7 @@ MapStatement =
     %s"map" SP Identifier [Mixins] *WS MapMembers
 
 MapMembers =
-    "{" *WS MapKey WS MapValue *WS "}"
+    "{" *WS [MapKey / MapValue / (MapKey WS MapValue)] *WS "}"
 
 MapKey =
     [TraitStatements] (ElidedMapKey / ExplicitMapKey)

modules/formatter/src/parsers/SmithyTraitParser.scala

@@ -21,7 +21,7 @@ import smithytranslate.formatter.ast.SmithyTraitBodyValue.{
   NodeValueCase,
   SmithyTraitStructureCase
 }
-import smithytranslate.formatter.parsers.WhitespaceParser.{br, sp, ws}
+import smithytranslate.formatter.parsers.WhitespaceParser.{sp, ws}
 import smithytranslate.formatter.ast.*
 import smithytranslate.formatter.parsers.NodeParser.{
   node_object_key,
@@ -61,16 +61,16 @@ object SmithyTraitParser {
       TraitStatements(traits, ws)
     }
   val apply_singular: Parser[ApplyStatementSingular] =
-    (Parser.string("apply") *> (ws ~ shape_id ~ ws ~ strait ~ br)).map {
-      case ((((a, b), c), d), e) => ApplyStatementSingular(a, b, c, d, e)
+    (Parser.string("apply") *> (ws ~ shape_id ~ ws ~ strait)).map {
+      case (((a, b), c), d) => ApplyStatementSingular(a, b, c, d)
     }
 
   val apply_block: Parser[ApplyStatementBlock] =
-    Parser.string(
+    (Parser.string(
       "apply"
-    ) *> ((sp *> shape_id ~ ws <* openCurly) ~ trait_statements ~ (closeCurly *> br))
-      .map { case (((a, b), c), d) =>
-        ApplyStatementBlock(a, b, c, d)
+    ) *> ((sp *> shape_id ~ ws <* openCurly) ~ trait_statements <* closeCurly))
+      .map { case ((a, b), c) =>
+        ApplyStatementBlock(a, b, c)
       }
   val apply_statement: Parser[ApplyStatement] =
     apply_block.backtrack.eitherOr(apply_singular).map(ApplyStatement)
@@ -97,11 +97,11 @@ TraitStructureKvp =
     NodeObjectKey *WS ":" *WS NodeValue
 
 ApplyStatement =
-    (ApplyStatementSingular / ApplyStatementBlock)
+    ApplyStatementSingular / ApplyStatementBlock
 
 ApplyStatementSingular =
-    %s"apply" WS ShapeId WS Trait BR
+    %s"apply" WS ShapeId WS Trait
 
 ApplyStatementBlock =
-    %s"apply" SP ShapeId WS "{" TraitStatements "}" BR
+    %s"apply" SP ShapeId WS "{" TraitStatements "}"
  */

modules/formatter/src/parsers/package.scala

@@ -18,17 +18,20 @@ import cats.parse.Parser
 
 package object parsers {
 
+  private val tabChar = 0x09.toChar
+  val tab: Parser[Unit] = Parser.char(tabChar)
   val openCurly: Parser[Unit] = Parser.char('{')
   val closeCurly: Parser[Unit] = Parser.char('}')
   val openSquare: Parser[Unit] = Parser.char('[')
   val colon: Parser[Unit] = Parser.char(':')
   val closeSquare: Parser[Unit] = Parser.char(']')
   val escapeChars: List[Char] =
-    List('\\', '"', '\'', 'b', 'f', 'n', 'r', 't', '/')
-  val quotable0: List[Char] = (0x20.toChar to 0x21.toChar).toList
-  val quotable1: List[Char] = (0x23.toChar to 0x5b.toChar).toList
-  val quotable2: List[Char] = (0x5d.toChar to 0x10ffff.toChar).toList
-  val allQuotable: List[Char] = quotable0 ++ quotable1 ++ quotable2
+    List('\\', '"', 'b', 'f', 'n', 'r', 't', '/')
+  private val quotable0: List[Char] = (0x20.toChar to 0x21.toChar).toList
+  private val quotable1: List[Char] = (0x23.toChar to 0x5b.toChar).toList
+  private val quotable2: List[Char] = (0x5d.toChar to 0x10ffff.toChar).toList
+  val allQuotable: List[Char] =
+    quotable0 ++ quotable1 ++ quotable2 ++ List(tabChar)
   val op: List[Char] = List('+', '-')
   val escape: Parser[Unit] = Parser.char(0x5c)
   val zero: Parser[Unit] = Parser.char('0')