feat: CDDL grammar correction for RFC8610

hermes/crates/cbork/cddl-parser/src/grammar/cddl_test.pest

@@ -5,38 +5,56 @@
 
 // cspell: words intfloat hexfloat
 
-/// Test Expression for the S Rule.
-S_TEST = ${ SOI ~ S ~ EOI }
+/// Test Expression for the `group` Rule.
+group_TEST  = { SOI ~ group ~ EOI }
 
-/// Test Expression for the COMMENT Rule.
-COMMENT_TEST = { SOI ~ COMMENT* ~ EOI }
+/// Test Expression for the `grpchoice` Rule.
+grpchoice_TEST  = { SOI ~ grpchoice ~ EOI }
+
+/// Test Expression for the `grpent` Rule.
+grpent_TEST  = { SOI ~ grpent ~ EOI }
+
+/// Test Expression for the `memberkey` Rule.
+memberkey_TEST  = { SOI ~ memberkey ~ EOI }
+
+/// Test Expression for the `bareword` Rule.
+bareword_TEST  = { SOI ~ bareword ~ EOI }
 
-/// Test expression for the URL_BASE64 Rule.
-URL_BASE64_TEST = { SOI ~ URL_BASE64 ~ EOI }
+/// Test Expression for the `optcom` Rule.
+optcom_TEST  = { SOI ~ optcom ~ EOI }
+
+/// Test Expression for the `occur` Rule.
+occur_TEST  = { SOI ~ occur ~ EOI }
+
+/// Test Expression for the `S` Rule.
+S_TEST = { SOI ~ S ~ EOI }
+
+/// Test Expression for the `COMMENT` Rule.
+COMMENT_TEST = { SOI ~ COMMENT* ~ EOI }
 
-/// Test expression to the id Rule.
-id_TEST = ${ SOI ~ id ~ EOI}
+/// Test expression to the `id` Rule.
+id_TEST = { SOI ~ id ~ EOI}
 
-/// Test expression to the bytes Rule.
-bytes_TEST = ${ SOI ~ bytes ~ EOI}
+/// Test expression to the `bytes` Rule.
+bytes_TEST = { SOI ~ bytes ~ EOI}
 
-/// Test expression to the text Rule.
-text_TEST = ${ SOI ~ text ~ EOI}
+/// Test expression to the `text` Rule.
+text_TEST = { SOI ~ text ~ EOI}
 
-/// Test expression to the uint Rule.
-uint_TEST = ${ SOI ~ uint ~ EOI}
+/// Test expression to the `uint` Rule.
+uint_TEST = { SOI ~ uint ~ EOI}
 
-/// Test expression to the int Rule.
-int_TEST = ${ SOI ~ int ~ EOI}
+/// Test expression to the `int` Rule.
+int_TEST = { SOI ~ int ~ EOI}
 
-/// Test expression to the intfloat Rule.
-intfloat_TEST = ${ SOI ~ intfloat ~ EOI}
+/// Test expression to the `intfloat` Rule.
+intfloat_TEST = { SOI ~ intfloat ~ EOI}
 
-/// Test expression to the hexfloat Rule.
-hexfloat_TEST = ${ SOI ~ hexfloat ~ EOI}
+/// Test expression to the `hexfloat` Rule.
+hexfloat_TEST = { SOI ~ hexfloat ~ EOI}
 
-/// Test expression to the number Rule.
-number_TEST = ${ SOI ~ number ~ EOI}
+/// Test expression to the `number` Rule.
+number_TEST = { SOI ~ number ~ EOI}
 
-/// Test expression to the value Rule.
-value_TEST = ${ SOI ~ value ~ EOI}
+/// Test expression to the `value` Rule.
+value_TEST = { SOI ~ value ~ EOI}

hermes/crates/cbork/cddl-parser/src/grammar/rfc_8610.pest

@@ -5,73 +5,75 @@
 // cspell: words genericarg rangeop ctlop grpchoice memberkey bareword hexfloat intfloat
 // cspell: words SCHAR BCHAR PCHAR SESC FFFD Characterset Visiable
 
-cddl = {
+cddl = ${
     SOI
-    ~ S ~ rule+
+    ~ S ~ (rule ~ S)+
     ~ EOI
 }
 
-rule = {
-    ( typename ~ assignt ~ type)
-    | ( groupname ~ assigng ~ grpent)
+// -----------------------------------------------------------------------------
+// Rules
+rule = ${
+    (typename ~ S ~ assignt ~ S ~ type)
+    | (groupname ~ S ~ assigng ~ S ~ grpent)
 }
 
-typename = ${ id ~ genericparm? }
-groupname = ${ id ~ genericparm? }
+typename = { id }
+groupname = { id }
 
 assignt = { "=" | "/=" }
 assigng = { "=" | "//=" }
 
-genericparm = { "<" ~ id ~ ( "," ~ id )* ~ ">" }
-genericarg = { "<" ~ type1 ~ ( "," ~ type1)* ~ ">" }
-
-type = { type1 ~ ( S ~ "/" ~ type1)* }
-
-type1 = { type2 ~ ( S ~ ( rangeop | ctlop ) ~ type2)? }
+genericparm = ${ "<" ~ S ~ id ~ S ~ ("," ~ S ~ id ~ S)* ~ ">" }
+genericarg = ${ "<" ~ S ~ type1 ~ S ~ ("," ~ S ~ type1 ~ S)* ~ ">" }
 
-typename_arg = ${ typename ~ genericarg? }
-groupname_arg = ${ groupname ~ genericarg? }
+// -----------------------------------------------------------------------------
+// Type Declaration
+type = ${ type1 ~ (S ~ "/" ~ S ~ type1)* }
 
-tag6 = ${ "#" ~ "6" ~ ("." ~ uint)? ~ "(" ~ S ~ type ~ S ~ ")" }
-tag_generic = ${ "#" ~ ASCII_DIGIT ~ ("." ~ uint)? }
+type1 = ${ type2 ~ (S ~ (rangeop | ctlop) ~ S ~ type2)? }
 
-type2 = {
+type2 = ${
     value
-    | typename_arg
-    | ( "(" ~ type ~ ")" )
-    | ( "{" ~ group ~ "}" )
-    | ( "[" ~ group ~ "]" )
-    | ( "~" ~ typename_arg )
-    | ( "&" ~ "(" ~ group ~ ")" )
-    | ( "&" ~ groupname_arg )
-    | tag6
-    | tag_generic
+    | typename ~ genericarg?
+    | ("(" ~ S ~ type ~ S ~ ")")
+    | ("{" ~ S ~ group ~ S ~ "}")
+    | ("[" ~ S ~ group ~ S ~ "]")
+    | ("~" ~ S ~ typename ~ genericarg?)
+    | ("&" ~ S ~ "(" ~ S ~ group ~ S ~ ")")
+    | ("&" ~ S ~ groupname ~ genericarg?)
+    | ("#" ~ "6" ~ ("." ~ uint)? ~ "(" ~ S ~ type ~ S ~ ")")
+    | ("#" ~ ASCII_DIGIT ~ ("." ~ uint)?)
     | "#"
 }
 
 rangeop = { "..." | ".." }
 ctlop = ${ "." ~ id }
 
-group = { grpchoice ~ ( S ~ "//" ~ grpchoice)* }
+// -----------------------------------------------------------------------------
+// Group Elements
+group = ${ grpchoice ~ (S ~ "//" ~ S ~ grpchoice)* }
 
-grpchoice = { ( grpent ~ ","? )* }
+grpchoice = ${ (grpent ~ optcom)* }
 
 grpent = ${
-    ( (occur ~ S)? ~ (memberkey ~ S)? ~ type )
-    | ( (occur ~ S)? ~ groupname ~ genericarg? )
-    | ( (occur ~ S)? ~ "(" ~ S ~ group ~ S ~ ")" )
+    ((occur ~ S)? ~ (memberkey ~ S)? ~ type)
+    | ((occur ~ S)? ~ groupname ~ genericarg?)
+    | ((occur ~ S)? ~ "(" ~ S ~ group ~ S ~ ")")
 }
 
-memberkey = {
-    ( type1 ~ "^"? ~ "=>" )
-    | ( bareword ~ ":" )
-    | ( value ~ ":" )
+memberkey = ${
+    (type1 ~ S ~ ("^" ~ S)? ~ "=>")
+    | ((value | bareword) ~ S ~ ":")
 }
 
 bareword = { id }
 
+/// Optional Comma - Note eligible for producing pairs as this might be useful for linting
+optcom = { S ~ ("," ~ S)? }
+
 occur = {
-    ( uint? ~ "*" ~ uint? )
+    (uint? ~ "*" ~ uint?)
     | "+"
     | "?"
 }
@@ -82,7 +84,7 @@ occur = {
 /// All Literal Values
 value = { number | text | bytes }
 
-/// Literal Numbers  - A float if it has fraction or exponent; int otherwise
+/// Literal Numbers - A float if it has fraction or exponent; int otherwise
 number = { hexfloat | intfloat }
 
 /// Hex floats of the form -0x123.abc0p+12
@@ -103,58 +105,33 @@ int = ${ "-"? ~ uint }
 
 /// Unsigned Integers
 uint = ${
-    ( ASCII_NONZERO_DIGIT ~ ASCII_DIGIT* )
-    | ( "0x" ~ ASCII_HEX_DIGIT+ )
-    | ( "0b" ~ ASCII_BIN_DIGIT+ )
+    (ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*)
+    | ("0x" ~ ASCII_HEX_DIGIT+)
+    | ("0b" ~ ASCII_BIN_DIGIT+)
     | "0"
 }
 
 /// Literal Text
 text = ${ "\"" ~ SCHAR* ~ "\"" }
 
-/// Literal Bytes - Note CDDL Spec incorrectly defines b64''.
-bytes = ${ bytes_hex | bytes_b64 | bytes_text }
-bytes_hex = ${ "h" ~ "'" ~ HEX_PAIR* ~ "'" }
-bytes_b64 = ${ "b64" ~ "'" ~ URL_BASE64 ~ "'" }
-bytes_text = ${ "'" ~ BCHAR* ~ "'" }
+/// Literal Bytes.
+bytes = ${ bsqual? ~ "'" ~ BCHAR* ~ "'" }
+bsqual = { "h" | "b64" }
 
 // -----------------------------------------------------------------------------
 // Simple multiple character sequences
 
-/// identifier,  called the `name` in the CDDL spec.
-id = ${
-    group_socket |
-    type_socket |
-    name
-}
-
-/// Special form of a name that represents a Group Socket.
-group_socket = ${ "$$" ~ ( ( "-" | "." )* ~ NAME_END )* }
-/// Special form of a name that represents a Type Socket.
-type_socket = ${ "$" ~ ( ( "-" | "." )* ~ NAME_END )* }
-/// General form of a name.
-name = ${ NAME_START ~ ( ( "-" | "." )* ~ NAME_END )* }
-
-/// A pair of hex digits. (Must always have even numbers of hex digits.)
-HEX_PAIR = _{ S ~ ASCII_HEX_DIGIT ~ S ~ ASCII_HEX_DIGIT ~ S }
-
-/// Whitespace is allowed and is ignored.
-/// This token will keep the whitespace, so it will need to handled when converted to binary.
-URL_BASE64 = _{ S ~ ( URL_BASE64_ALPHA ~ S)* ~ URL_BASE64_PAD? }
+/// identifier, called the `name` in the CDDL spec.
+id = ${ NAME_START ~ (("-" | ".")* ~ NAME_END)* }
 
 
 // -----------------------------------------------------------------------------
 // Characters, Whitespace and Comments
 
 S = _{ WHITESPACE* }
-WHITESPACE = _{ " " | "\t" | NEWLINE }
+WHITESPACE = _{ " " | "\t" | COMMENT | NEWLINE }
 COMMENT = _{ ";" ~ (PCHAR | "\t")* ~ NEWLINE }
 
-// URL Base64 Characterset.
-URL_BASE64_ALPHA = _{ ASCII_ALPHA | ASCII_DIGIT |  "-" | "_"  }
-// Optional Padding that goes at the end of Base64.
-URL_BASE64_PAD = _{ "~" }
-
 // Identifier Name Character sets.
 
 /// A name can start with an alphabetic character (including "@", "_", "$")
@@ -172,12 +149,12 @@ PCHAR = _{ ASCII_VISIBLE | UNICODE_CHAR }
 SCHAR = _{ SCHAR_ASCII_VISIBLE | UNICODE_CHAR | SESC }
 
 /// The set of characters valid for a byte string.
-BCHAR = _{ BCHAR_ASCII_VISIBLE | UNICODE_CHAR | SESC | NEWLINE }
+BCHAR = _{ '\u{20}'..'\u{26}' | '\u{28}'..'\u{5B}' | '\u{5D}'..'\u{10FFFD}' | SESC | NEWLINE }
 
 /// Escaping code to allow invalid characters to be used in text or byte strings.
 SESC = ${ "\\" ~ (ASCII_VISIBLE | UNICODE_CHAR) }
 
-/// All Visiable Ascii characters.
+/// All Visible Ascii characters.
 ASCII_VISIBLE = _{ ' '..'~' }
 
 /// Ascii subset valid for text strings.
@@ -187,4 +164,4 @@ SCHAR_ASCII_VISIBLE = _{ ' '..'!' | '#'..'[' | ']'..'~' }
 BCHAR_ASCII_VISIBLE = _{ ' '..'&' | '('..'[' | ']'..'~' }
 
 /// Valid non ascii unicode Characters
-UNICODE_CHAR = _{ '\u{80}'..'\u{10FFFD}' }
+UNICODE_CHAR = _{ '\u{80}'..'\u{10FFFD}' }

hermes/crates/cbork/cddl-parser/src/lib.rs

@@ -20,12 +20,12 @@ pub mod rfc_8610 {
     pub struct RFC8610Parser;
 }
 
-pub mod rfc_9615 {
+pub mod rfc_9165 {
     pub use pest::Parser;
 
     #[derive(pest_derive::Parser)]
     #[grammar = "grammar/rfc_8610.pest"]
-    #[grammar = "grammar/rfc_9615.pest"]
+    #[grammar = "grammar/rfc_9165.pest"]
     pub struct RFC8610Parser;
 }
 
@@ -34,7 +34,7 @@ pub mod cddl {
 
     #[derive(pest_derive::Parser)]
     #[grammar = "grammar/rfc_8610.pest"]
-    #[grammar = "grammar/rfc_9615.pest"]
+    #[grammar = "grammar/rfc_9165.pest"]
     #[grammar = "grammar/cddl_modules.pest"]
     pub struct RFC8610Parser;
 }
@@ -45,7 +45,7 @@ pub mod cddl_test {
     // Parser with DEBUG rules. These rules are only used in tests.
     #[derive(pest_derive::Parser)]
     #[grammar = "grammar/rfc_8610.pest"]
-    #[grammar = "grammar/rfc_9615.pest"]
+    #[grammar = "grammar/rfc_9165.pest"]
     #[grammar = "grammar/cddl_modules.pest"]
     #[grammar = "grammar/cddl_test.pest"] // Ideally this would only be used in tests.
     pub struct CDDLTestParser;
@@ -55,9 +55,9 @@ pub mod cddl_test {
 pub enum Extension {
     /// RFC8610 ONLY limited parser.
     RFC8610Parser,
-    /// RFC8610 and RFC9615 limited parser.
-    RFC9615Parser,
-    /// RFC8610, RFC9615, and CDDL modules.
+    /// RFC8610 and RFC9165 limited parser.
+    RFC9165Parser,
+    /// RFC8610, RFC9165, and CDDL modules.
     CDDLParser,
 }
 
@@ -68,7 +68,7 @@ pub const POSTLUDE: &str = include_str!("grammar/postlude.cddl");
 #[derive(Debug)]
 pub enum AST<'a> {
     RFC8610(Pairs<'a, rfc_8610::Rule>),
-    RFC9615(Pairs<'a, rfc_9615::Rule>),
+    RFC9165(Pairs<'a, rfc_9165::Rule>),
     CDDL(Pairs<'a, cddl::Rule>),
 }
 
@@ -77,8 +77,8 @@ pub enum AST<'a> {
 pub enum CDDLErrorType {
     /// An error related to RFC 8610 extension.
     RFC8610(Error<rfc_8610::Rule>),
-    /// An error related to RFC 9615 extension.
-    RFC9615(Error<rfc_9615::Rule>),
+    /// An error related to RFC 9165 extension.
+    RFC9165(Error<rfc_9165::Rule>),
     /// An error related to CDDL modules extension.
     CDDL(Error<cddl::Rule>),
 }
@@ -126,10 +126,10 @@ pub fn parse_cddl<'a>(
                 .map(AST::RFC8610)
                 .map_err(CDDLErrorType::RFC8610)
         },
-        Extension::RFC9615Parser => {
-            rfc_9615::RFC8610Parser::parse(rfc_9615::Rule::cddl, input)
-                .map(AST::RFC9615)
-                .map_err(CDDLErrorType::RFC9615)
+        Extension::RFC9165Parser => {
+            rfc_9165::RFC8610Parser::parse(rfc_9165::Rule::cddl, input)
+                .map(AST::RFC9165)
+                .map_err(CDDLErrorType::RFC9165)
         },
         Extension::CDDLParser => {
             cddl::RFC8610Parser::parse(cddl::Rule::cddl, input)
@@ -139,9 +139,6 @@ pub fn parse_cddl<'a>(
     };
 
     result.map(Box::new).map_err(|e| {
-        println!("{e:?}");
-        println!("{e}");
-
         Box::new(CDDLError::from(e))
     })
 }