From e8e42ef2b63a2fbb13baf1a0b8070d3b9da0dba7 Mon Sep 17 00:00:00 2001 From: Dominik Nakamura Date: Tue, 20 Feb 2024 10:17:45 +0900 Subject: [PATCH] feat: track keywords, punctuation and delimiters In addition to the more valuable parts of a schema that describe the data structures, now the remaining pieces that make up a schema are tracked as well. That means characters like commas, semicolons, open/close braces and others are collected so their exact location in the source file can be used by the LSP and other components. A few places still don't track this information as they need some further refactoring first, but should be doing so in the following commits. --- CHANGELOG.md | 33 +++ crates/mabo-compiler/src/resolve/mod.rs | 18 +- crates/mabo-compiler/src/simplify.rs | 20 +- crates/mabo-compiler/src/validate/generics.rs | 18 +- crates/mabo-compiler/src/validate/ids.rs | 4 +- crates/mabo-compiler/src/validate/names.rs | 4 +- crates/mabo-compiler/src/validate/tuples.rs | 18 +- crates/mabo-derive/src/debug.rs | 22 +- .../mabo-lsp/src/handlers/document_symbols.rs | 4 +- .../mabo-lsp/src/handlers/semantic_tokens.rs | 149 +++++++++-- crates/mabo-parser/src/lib.rs | 210 +++++++++++----- crates/mabo-parser/src/parser/aliases.rs | 49 ++-- crates/mabo-parser/src/parser/consts.rs | 57 +++-- crates/mabo-parser/src/parser/enums.rs | 58 +++-- crates/mabo-parser/src/parser/fields.rs | 70 ++++-- crates/mabo-parser/src/parser/imports.rs | 44 ++-- crates/mabo-parser/src/parser/modules.rs | 47 ++-- crates/mabo-parser/src/parser/structs.rs | 35 +-- crates/mabo-parser/src/parser/types.rs | 144 +++++++---- crates/mabo-parser/src/token.rs | 234 ++++++++++++++++++ .../parser__parse@alias_basic.mabo.snap | 3 + .../parser__parse@attribute_multi.mabo.snap | 1 + .../parser__parse@attribute_single.mabo.snap | 1 + .../parser__parse@attribute_unit.mabo.snap | 1 + .../parser__parse@attributes.mabo.snap | 1 + .../parser__parse@attributes_min_ws.mabo.snap | 1 + .../parser__parse@const_basic.mabo.snap | 24 ++ .../parser__parse@const_string.mabo.snap | 16 ++ .../parser__parse@enum_basic.mabo.snap | 25 ++ .../parser__parse@enum_generics.mabo.snap | 29 +++ .../parser__parse@enum_many_ws.mabo.snap | 25 ++ .../parser__parse@enum_min_ws.mabo.snap | 30 +++ .../parser__parse@import_basic.mabo.snap | 4 + .../snapshots/parser__parse@mixed.mabo.snap | 151 +++++++++++ .../parser__parse@module_basic.mabo.snap | 20 ++ .../parser__parse@optional_ids.mabo.snap | 55 ++++ .../parser__parse@schema_basic.mabo.snap | 35 +++ .../parser__parse@struct_basic.mabo.snap | 10 + .../parser__parse@struct_generics.mabo.snap | 12 + .../parser__parse@struct_many_ws.mabo.snap | 15 ++ .../parser__parse@struct_min_ws.mabo.snap | 13 + .../parser__parse@struct_tuple.mabo.snap | 6 + .../parser__parse@types_basic.mabo.snap | 89 +++++++ .../parser__parse@types_generic.mabo.snap | 51 ++++ .../parser__parse@types_nested.mabo.snap | 11 + .../parser__parse@types_non_zero.mabo.snap | 81 ++++++ .../parser__parse@types_ref.mabo.snap | 31 +++ .../parser__print@enum_min_ws.mabo.snap | 4 +- .../parser__print@struct_min_ws.mabo.snap | 2 +- .../parser__print@types_generic.mabo.snap | 2 +- vscode-extension/package.json | 72 ++++++ 51 files changed, 1725 insertions(+), 334 deletions(-) create mode 100644 crates/mabo-parser/src/token.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fca0d8..02bfb13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,13 @@ All notable changes to this project will be documented in this file. > > This is currently used to log the location in the LPS when loading a > folder and searching for Mabo projects in it. +- _lsp_: Support all official position encodings ([e6d4884](https://github.com/dnaka91/mabo/commit/e6d48848a6e23bf5fd1c43a905ea4904fa0adf9a)) + > Until now the server always assumed UTF-16, as that is the default in + > Visual Studio Code, but other editors like Neovim might ask for a + > different encoding. + > + > This now allows to run the LSP server with any of the official encodings + > UTF-8, UTF-16 and UTF-32. - _parser_: Add `simd` feature to the parser ([443c32c](https://github.com/dnaka91/mabo/commit/443c32c93622764a89cd3f0682ef8b1c6a6723dc)) > The `winnow` crate has a `simd` feature which enables some extra > performance improvements. This might limit possible target platforms for @@ -111,6 +118,7 @@ All notable changes to this project will be documented in this file. - _vscode_: Provide command to restart the LSP server ([21e5001](https://github.com/dnaka91/mabo/commit/21e500105b651ecede36495b98d3ad61ac1466a1)) > Although mostly useful during development, add a custom command to the > VSCode extension, which allows to restart the LSP server. +- _vscode_: Extend metadata for the vscode extension ([1dd3ca8](https://github.com/dnaka91/mabo/commit/1dd3ca83417535d1e91a478f209f10de4f032b97)) - Implement encoding in the Rust codegen ([2787b51](https://github.com/dnaka91/mabo/commit/2787b51c04803311bf5ca3160b37e7db31d5a8ea)) - Simplify encoding logic ([e8205bc](https://github.com/dnaka91/mabo/commit/e8205bcb6749fce6dd1f56ede38128076820bffd)) - Implement basic decoding logic ([f67c572](https://github.com/dnaka91/mabo/commit/f67c57220ac2c57961bf54f7f47bca467d3fb20b)) @@ -257,6 +265,9 @@ All notable changes to this project will be documented in this file. > first empty line is found, instead of simply taking the first line only. > > This fixes potential cut-off of the overview item docs. +- _doc_: Fix broken PR links in the changelog ([359f02f](https://github.com/dnaka91/mabo/commit/359f02f284e9ac6d46f7dcbf22abd094a99a9123)) + > Due to how repo placeholders were used in git-cliff, those were mangled + > and then later not properly replaced with the URL anymore. - _lsp_: Remove closed documents from the state ([a84018c](https://github.com/dnaka91/mabo/commit/a84018c6206cb17c00284bc792901fecf6785700)) > To avoid growing the list of open files indefinitely, remove them from > the state of the server when closed in the editor. @@ -310,6 +321,13 @@ All notable changes to this project will be documented in this file. > The calculation for required bytes of a field ID did not take the > recently added field encoding into account, which could potentially > result in too small values. +- Expand tmLanguage regexes for tuples and arrays ([b589eac](https://github.com/dnaka91/mabo/commit/b589eac40d2d83ef5a5eb224229161fc177d7fe3)) + > The tuples and arrays were correctly parsed for individual types, but + > the field regex didn't include possible token which made it fail to + > include those types in them matching. +- Update winnow to fix broken integer parsing ([1f1790a](https://github.com/dnaka91/mabo/commit/1f1790a188db58b01e50804566d86265b919138b)) + > Version 0.6 introduced a bug that caused the literal `0` not to be + > parsed as valid integer anymore. This was fixed with 0.6.1. ### 📚 Documentation @@ -351,6 +369,7 @@ All notable changes to this project will be documented in this file. > Remove the sections from the ideas sections that have already been > implemented recently. - _parser_: Add missing doc for new field ([7f4ca98](https://github.com/dnaka91/mabo/commit/7f4ca98c236a41e505bcaa70a2df5fc3aae85b7a)) +- _vscode_: Add dedicated readme for the extension ([38af273](https://github.com/dnaka91/mabo/commit/38af273445a37dc56e04a90c85f0f57ae5621a1a)) - Generate more stylish changelog ([5319fb3](https://github.com/dnaka91/mabo/commit/5319fb3417a830042e7bc220fe283046923da349)) - Add changelog ([5b2a15c](https://github.com/dnaka91/mabo/commit/5b2a15cad70e53c6c39a93c395fbe8f80382ae56)) - Update flatbuffers homepage in the book ([c469e4e](https://github.com/dnaka91/mabo/commit/c469e4e966cfb3866d08369f813eb999a4c3032d)) @@ -394,10 +413,19 @@ All notable changes to this project will be documented in this file. > The current schema generated for benchmarks on large schemas didn't > generate any definitions that use type references. Therefore, the > benchmark didn't give good insight on type resolution timing. +- Use a faster hasher in compiler and LSP ([4eb37aa](https://github.com/dnaka91/mabo/commit/4eb37aaafc4da7a8b73b6aae31c38672cb8554c9)) + > By applying the same hasher as in Rust's compiler, performance can be + > improved wherever a hash map or set is used. + > + > This currently has the most visible impact on the compiler's validation + > step as it internally makes heavy use of hash maps. ### 🚜 Refactor - _compiler_: Simplify some type resolution steps ([ae1cf34](https://github.com/dnaka91/mabo/commit/ae1cf344feef941dc7f7afe6e8dca9e30c92b7d5)) +- _go_: Simplify indenting logic ([7425a11](https://github.com/dnaka91/mabo/commit/7425a118030d080acd025bbdbc7a2bc4ccc80058)) + > Use a dedicated type that implements the `Display` trait, which allows + > for a much simpler way of expressing and forwarding indents. - _lsp_: Avoid unwraps and properly handle errors ([bcbb016](https://github.com/dnaka91/mabo/commit/bcbb016a1180a38311cbaa2709979494c0b2eb77)) - _lsp_: Move logging logic into its own module ([ef1139b](https://github.com/dnaka91/mabo/commit/ef1139be9290846ccd197c4bbeeddca48ffcff50)) - _lsp_: Replace tower-lsp with lsp-server ([51645a8](https://github.com/dnaka91/mabo/commit/51645a88c9b9c0d3e13a47da429ab9f7ef28d67f)) @@ -491,6 +519,7 @@ All notable changes to this project will be documented in this file. - _ci_: Setup GitHub Actions to deploy the book ([cf24bbb](https://github.com/dnaka91/mabo/commit/cf24bbb0e55955e37233f9719f046bb482a8d712)) - _ci_: Improve path filters for book deployment ([8b79f6f](https://github.com/dnaka91/mabo/commit/8b79f6f03787edde0db881ab0e61b6e163924709)) - _ci_: Create .nojekyll marker file for the book ([b51681f](https://github.com/dnaka91/mabo/commit/b51681f377640421249d9d9e180446f8ae16142a)) +- _doc_: Fix ambiguous links in the Rust docs ([e5d73fe](https://github.com/dnaka91/mabo/commit/e5d73fe68759aab1721f74e9845c57ea74fbd911)) - _lsp_: Remove duplicate log message ([e8cfa08](https://github.com/dnaka91/mabo/commit/e8cfa08e0798c924c61c555fc74359ad2c660bb0)) - Initial commit ([5eb2f2b](https://github.com/dnaka91/mabo/commit/5eb2f2b9687146363974ea645de22a8441e890a1)) - Update checkout action to v4 ([4d753d8](https://github.com/dnaka91/mabo/commit/4d753d8b30ef3ee7d7e463fb2e7f594aee86d8e7)) @@ -509,5 +538,9 @@ All notable changes to this project will be documented in this file. - Fix Just task for link checking ([24e1520](https://github.com/dnaka91/mabo/commit/24e15209799f6285ca3a55f7145556145cb61f30)) - Improve readme with badges and project status ([069f788](https://github.com/dnaka91/mabo/commit/069f7882f9abd1d14b697b38314a5db2976b895a)) - Update snapshots after several Go codegen fixes ([b38e1ca](https://github.com/dnaka91/mabo/commit/b38e1cad5b861e59774a03fb5768d2284376787e)) +- Bump MSRV to 1.76 and update dependencies ([554b05b](https://github.com/dnaka91/mabo/commit/554b05b8799abaaa06caa73260882d22ba4856f2)) + > Deprecations in `winnow` as well as increases in MSRVs in the + > dependencies. Bumping the MSRV to the very latest Rust version as there + > are no MSRV promises as of now and `clap` bumped their MSRV. diff --git a/crates/mabo-compiler/src/resolve/mod.rs b/crates/mabo-compiler/src/resolve/mod.rs index d869165..6f850a9 100644 --- a/crates/mabo-compiler/src/resolve/mod.rs +++ b/crates/mabo-compiler/src/resolve/mod.rs @@ -352,12 +352,12 @@ pub(crate) fn resolve_module_types<'a>( module: &'a Module<'_>, ) { match fields { - Fields::Named(named) => { + Fields::Named(_, named) => { for field in named { resolve(missing, &field.ty, generics, module); } } - Fields::Unnamed(unnamed) => { + Fields::Unnamed(_, unnamed) => { for field in unnamed { resolve(missing, &field.ty, generics, module); } @@ -470,20 +470,20 @@ fn visit_externals<'a>(value: &'a Type<'_>, visit: &mut impl FnMut(&'a ExternalT | DataType::StringRef | DataType::Bytes | DataType::BytesRef - | DataType::NonZero(_) + | DataType::NonZero(_, _, _) | DataType::BoxString | DataType::BoxBytes => {} - DataType::Vec(ty) - | DataType::HashSet(ty) - | DataType::Option(ty) - | DataType::Array(ty, _) => { + DataType::Vec(_, _, ty) + | DataType::HashSet(_, _, ty) + | DataType::Option(_, _, ty) + | DataType::Array(_, ty, _, _) => { visit_externals(ty, visit); } - DataType::HashMap(kv) => { + DataType::HashMap(_, _, _, kv) => { visit_externals(&kv.0, visit); visit_externals(&kv.1, visit); } - DataType::Tuple(types) => { + DataType::Tuple(_, types) => { for ty in types { visit_externals(ty, visit); } diff --git a/crates/mabo-compiler/src/simplify.rs b/crates/mabo-compiler/src/simplify.rs index 8990e10..b8a05a9 100644 --- a/crates/mabo-compiler/src/simplify.rs +++ b/crates/mabo-compiler/src/simplify.rs @@ -383,7 +383,7 @@ fn simplify_fields<'a>(item: &'a mabo_parser::Fields<'_>) -> Fields<'a> { let mut id_gen = IdGenerator::new(); match item { - mabo_parser::Fields::Named(named) => Fields { + mabo_parser::Fields::Named(_, named) => Fields { source: item, fields: named .iter() @@ -397,7 +397,7 @@ fn simplify_fields<'a>(item: &'a mabo_parser::Fields<'_>) -> Fields<'a> { .collect(), kind: FieldKind::Named, }, - mabo_parser::Fields::Unnamed(unnamed) => Fields { + mabo_parser::Fields::Unnamed(_, unnamed) => Fields { source: item, fields: unnamed .iter() @@ -439,19 +439,21 @@ fn simplify_type<'a>(item: &'a mabo_parser::Type<'_>) -> Type<'a> { mabo_parser::DataType::StringRef => Type::StringRef, mabo_parser::DataType::Bytes => Type::Bytes, mabo_parser::DataType::BytesRef => Type::BytesRef, - mabo_parser::DataType::Vec(ref ty) => Type::Vec(simplify_type(ty).into()), - mabo_parser::DataType::HashMap(ref kv) => { + mabo_parser::DataType::Vec(_, _, ref ty) => Type::Vec(simplify_type(ty).into()), + mabo_parser::DataType::HashMap(_, _, _, ref kv) => { Type::HashMap((simplify_type(&kv.0), simplify_type(&kv.1)).into()) } - mabo_parser::DataType::HashSet(ref ty) => Type::HashSet(simplify_type(ty).into()), - mabo_parser::DataType::Option(ref ty) => Type::Option(simplify_type(ty).into()), - mabo_parser::DataType::NonZero(ref ty) => Type::NonZero(simplify_type(ty).into()), + mabo_parser::DataType::HashSet(_, _, ref ty) => Type::HashSet(simplify_type(ty).into()), + mabo_parser::DataType::Option(_, _, ref ty) => Type::Option(simplify_type(ty).into()), + mabo_parser::DataType::NonZero(_, _, ref ty) => Type::NonZero(simplify_type(ty).into()), mabo_parser::DataType::BoxString => Type::BoxString, mabo_parser::DataType::BoxBytes => Type::BoxBytes, - mabo_parser::DataType::Tuple(ref types) => { + mabo_parser::DataType::Tuple(_, ref types) => { Type::Tuple(types.iter().map(|ty| simplify_type(ty)).collect()) } - mabo_parser::DataType::Array(ref ty, size) => Type::Array(simplify_type(ty).into(), size), + mabo_parser::DataType::Array(_, ref ty, _, size) => { + Type::Array(simplify_type(ty).into(), size) + } mabo_parser::DataType::External(ref ty) => Type::External(ExternalType { path: ty.path.iter().map(mabo_parser::Name::get).collect(), name: ty.name.get(), diff --git a/crates/mabo-compiler/src/validate/generics.rs b/crates/mabo-compiler/src/validate/generics.rs index d08b522..07da0b6 100644 --- a/crates/mabo-compiler/src/validate/generics.rs +++ b/crates/mabo-compiler/src/validate/generics.rs @@ -114,7 +114,7 @@ fn validate_duplicate_generics(value: &Generics<'_>) -> Result<(), DuplicateGene /// field. fn validate_field_generics(value: &Fields<'_>, unvisited: &mut FxHashMap<&str, Span>) { match &value { - Fields::Named(named) => { + Fields::Named(_, named) => { for field in named { visit_externals(&field.ty, &mut |external| { if external.path.is_empty() && external.generics.is_empty() { @@ -123,7 +123,7 @@ fn validate_field_generics(value: &Fields<'_>, unvisited: &mut FxHashMap<&str, S }); } } - Fields::Unnamed(unnamed) => { + Fields::Unnamed(_, unnamed) => { for field in unnamed { visit_externals(&field.ty, &mut |external| { if external.path.is_empty() && external.generics.is_empty() { @@ -157,18 +157,18 @@ fn visit_externals(value: &Type<'_>, visit: &mut impl FnMut(&ExternalType<'_>)) | DataType::StringRef | DataType::Bytes | DataType::BytesRef - | DataType::NonZero(_) + | DataType::NonZero(_, _, _) | DataType::BoxString | DataType::BoxBytes => {} - DataType::Vec(ty) - | DataType::HashSet(ty) - | DataType::Option(ty) - | DataType::Array(ty, _) => visit_externals(ty, visit), - DataType::HashMap(kv) => { + DataType::Vec(_, _, ty) + | DataType::HashSet(_, _, ty) + | DataType::Option(_, _, ty) + | DataType::Array(_, ty, _, _) => visit_externals(ty, visit), + DataType::HashMap(_, _, _, kv) => { visit_externals(&kv.0, visit); visit_externals(&kv.1, visit); } - DataType::Tuple(types) => { + DataType::Tuple(_, types) => { for ty in types { visit_externals(ty, visit); } diff --git a/crates/mabo-compiler/src/validate/ids.rs b/crates/mabo-compiler/src/validate/ids.rs index afd6d28..e1469b3 100644 --- a/crates/mabo-compiler/src/validate/ids.rs +++ b/crates/mabo-compiler/src/validate/ids.rs @@ -132,7 +132,7 @@ pub(crate) fn validate_enum_ids(value: &Enum<'_>) -> Result<(), DuplicateId> { /// Ensure all field IDs of a struct or enum are unique. fn validate_field_ids(value: &Fields<'_>) -> Result<(), DuplicateFieldId> { match value { - Fields::Named(named) => { + Fields::Named(_, named) => { let mut visited = FxHashMap::with_capacity_and_hasher(named.len(), BuildHasherDefault::default()); let mut id_gen = IdGenerator::new(); @@ -154,7 +154,7 @@ fn validate_field_ids(value: &Fields<'_>) -> Result<(), DuplicateFieldId> { }) .map_or(Ok(()), Err)?; } - Fields::Unnamed(unnamed) => { + Fields::Unnamed(_, unnamed) => { let mut visited = FxHashMap::with_capacity_and_hasher(unnamed.len(), BuildHasherDefault::default()); let mut id_gen = IdGenerator::new(); diff --git a/crates/mabo-compiler/src/validate/names.rs b/crates/mabo-compiler/src/validate/names.rs index e03a460..5873ca6 100644 --- a/crates/mabo-compiler/src/validate/names.rs +++ b/crates/mabo-compiler/src/validate/names.rs @@ -105,7 +105,7 @@ pub(crate) fn validate_enum_names(value: &Enum<'_>) -> Result<(), DuplicateName> /// Ensure all field names of a struct or enum are unique. fn validate_field_names(value: &Fields<'_>) -> Result<(), DuplicateFieldName> { match value { - Fields::Named(named) => { + Fields::Named(_, named) => { let mut visited = FxHashMap::with_capacity_and_hasher(named.len(), BuildHasherDefault::default()); named @@ -121,7 +121,7 @@ fn validate_field_names(value: &Fields<'_>) -> Result<(), DuplicateFieldName> { }) .map_or(Ok(()), Err)?; } - Fields::Unnamed(_) | Fields::Unit => {} + Fields::Unnamed(_, _) | Fields::Unit => {} } Ok(()) diff --git a/crates/mabo-compiler/src/validate/tuples.rs b/crates/mabo-compiler/src/validate/tuples.rs index a97181e..e0f521a 100644 --- a/crates/mabo-compiler/src/validate/tuples.rs +++ b/crates/mabo-compiler/src/validate/tuples.rs @@ -49,10 +49,10 @@ pub(crate) fn validate_enum_tuples(value: &Enum<'_>) -> Result<(), TupleSize> { fn validate_field_tuples(value: &Fields<'_>) -> Result<(), TupleSize> { match value { - Fields::Named(named) => named + Fields::Named(_, named) => named .iter() .try_for_each(|field| validate_tuple_size(&field.ty)), - Fields::Unnamed(unnamed) => unnamed + Fields::Unnamed(_, unnamed) => unnamed .iter() .try_for_each(|field| validate_tuple_size(&field.ty)), Fields::Unit => Ok(()), @@ -106,18 +106,18 @@ fn visit_tuples( | DataType::StringRef | DataType::Bytes | DataType::BytesRef - | DataType::NonZero(_) + | DataType::NonZero(_, _, _) | DataType::BoxString | DataType::BoxBytes => Ok(()), - DataType::Vec(ty) - | DataType::HashSet(ty) - | DataType::Option(ty) - | DataType::Array(ty, _) => visit_tuples(ty, visit), - DataType::HashMap(kv) => { + DataType::Vec(_, _, ty) + | DataType::HashSet(_, _, ty) + | DataType::Option(_, _, ty) + | DataType::Array(_, ty, _, _) => visit_tuples(ty, visit), + DataType::HashMap(_, _, _, kv) => { visit_tuples(&kv.0, visit)?; visit_tuples(&kv.1, visit) } - DataType::Tuple(types) => { + DataType::Tuple(_, types) => { visit(types)?; types.iter().try_for_each(|ty| visit_tuples(ty, visit)) } diff --git a/crates/mabo-derive/src/debug.rs b/crates/mabo-derive/src/debug.rs index 4b7bdf3..27dba7c 100644 --- a/crates/mabo-derive/src/debug.rs +++ b/crates/mabo-derive/src/debug.rs @@ -111,11 +111,25 @@ fn expand_named_field(field: &Field) -> TokenStream { } } -fn expand_unnamed_field((i, _field): (usize, &Field)) -> TokenStream { - let ident = Ident::new(&format!("n{i}"), Span::call_site()); +fn expand_unnamed_field((i, field): (usize, &Field)) -> TokenStream { + if let Type::Tuple(tuple) = &field.ty { + let ident = Ident::new(&format!("n{i}"), Span::call_site()); + let index = tuple + .elems + .iter() + .enumerate() + .filter(|(_, ty)| !filter_span(ty)) + .map(|(j, _)| Literal::usize_unsuffixed(j)); + + quote! { + .field(&#(#ident.#index,)*) + } + } else { + let ident = Ident::new(&format!("n{i}"), Span::call_site()); - quote! { - .field(&#ident) + quote! { + .field(&#ident) + } } } diff --git a/crates/mabo-lsp/src/handlers/document_symbols.rs b/crates/mabo-lsp/src/handlers/document_symbols.rs index 9b44eee..213bb80 100644 --- a/crates/mabo-lsp/src/handlers/document_symbols.rs +++ b/crates/mabo-lsp/src/handlers/document_symbols.rs @@ -71,11 +71,11 @@ fn visit_variant(index: &Index, item: &Variant<'_>) -> Result { fn visit_fields(index: &Index, item: &Fields<'_>) -> Result> { match item { - Fields::Named(named) => named + Fields::Named(_, named) => named .iter() .map(|field| visit_named_field(index, field)) .collect(), - Fields::Unnamed(unnamed) => unnamed + Fields::Unnamed(_, unnamed) => unnamed .iter() .enumerate() .map(|(pos, field)| visit_unnamed_field(index, field, pos)) diff --git a/crates/mabo-lsp/src/handlers/semantic_tokens.rs b/crates/mabo-lsp/src/handlers/semantic_tokens.rs index 80af690..51ed740 100644 --- a/crates/mabo-lsp/src/handlers/semantic_tokens.rs +++ b/crates/mabo-lsp/src/handlers/semantic_tokens.rs @@ -1,8 +1,9 @@ use anyhow::{ensure, Result}; use lsp_types::{SemanticToken, SemanticTokenModifier, SemanticTokenType}; use mabo_parser::{ - Comment, Const, DataType, Definition, Enum, Fields, Generics, Id, Literal, LiteralValue, - Module, NamedField, Schema, Span, Spanned, Struct, Type, TypeAlias, UnnamedField, Variant, + token::Delimiter, Comment, Const, DataType, Definition, Enum, ExternalType, Fields, Generics, + Id, Import, Literal, LiteralValue, Module, NamedField, Schema, Span, Spanned, Struct, Type, + TypeAlias, UnnamedField, Variant, }; pub(crate) use self::{modifiers::TOKEN_MODIFIERS, types::TOKEN_TYPES}; @@ -42,7 +43,7 @@ define_semantic_token_types! { VARIABLE, PROPERTY, ENUM_MEMBER, - // KEYWORD, + KEYWORD, COMMENT, // STRING, NUMBER, @@ -55,6 +56,20 @@ define_semantic_token_types! { (BUILTIN_TYPE, "builtinType"), (IDENTIFIER, "identifier"), (TYPE_ALIAS, "typeAlias"), + + // Punctuation tokens + (COMMA, "comma"), + (COLON, "colon"), + (SEMICOLON, "semicolon"), + (POUND, "pound"), + (DOUBLE_COLON, "doubleColon"), + (EQUAL, "equal"), + + // Delimiter tokens + (BRACE, "brace"), + (BRACKET, "bracket"), + (PARENTHESIS, "parenthesis"), + (ANGLE, "angle"), } } @@ -202,23 +217,26 @@ impl<'a> Visitor<'a> { Definition::Enum(e) => self.visit_enum(e), Definition::TypeAlias(a) => self.visit_alias(a), Definition::Const(c) => self.visit_const(c), - Definition::Import(_i) => Ok(()), + Definition::Import(i) => self.visit_import(i), } } fn visit_module(&mut self, item: &Module<'_>) -> Result<()> { self.visit_comment(&item.comment)?; + self.add_span(&item.keyword, &types::KEYWORD, &[])?; self.add_span(&item.name, &types::NAMESPACE, &[modifiers::DECLARATION])?; + self.add_span(&item.brace.open(), &types::BRACE, &[])?; for def in &item.definitions { self.visit_definition(def)?; } - Ok(()) + self.add_span(&item.brace.close(), &types::BRACE, &[]) } fn visit_struct(&mut self, item: &Struct<'_>) -> Result<()> { self.visit_comment(&item.comment)?; + self.add_span(&item.keyword, &types::KEYWORD, &[])?; self.add_span(&item.name, &types::STRUCT, &[modifiers::DECLARATION])?; self.visit_generics(&item.generics)?; self.visit_fields(&item.fields) @@ -226,34 +244,44 @@ impl<'a> Visitor<'a> { fn visit_enum(&mut self, item: &Enum<'_>) -> Result<()> { self.visit_comment(&item.comment)?; + self.add_span(&item.keyword, &types::KEYWORD, &[])?; self.add_span(&item.name, &types::ENUM, &[modifiers::DECLARATION])?; self.visit_generics(&item.generics)?; + self.add_span(&item.brace.open(), &types::BRACE, &[])?; for variant in &item.variants { self.visit_variant(variant)?; } - Ok(()) + self.add_span(&item.brace.close(), &types::BRACE, &[]) } fn visit_variant(&mut self, item: &Variant<'_>) -> Result<()> { self.visit_comment(&item.comment)?; self.add_span(&item.name, &types::ENUM_MEMBER, &[modifiers::DECLARATION])?; self.visit_fields(&item.fields)?; - self.visit_id(&item.id) + self.visit_id(&item.id)?; + if let Some(comma) = &item.comma { + self.add_span(comma, &types::COMMA, &[])?; + } + Ok(()) } fn visit_fields(&mut self, item: &Fields<'_>) -> Result<()> { match item { - Fields::Named(named) => { + Fields::Named(brace, named) => { + self.add_span(&brace.open(), &types::BRACE, &[])?; for field in named { self.visit_named_field(field)?; } + self.add_span(&brace.close(), &types::BRACE, &[])?; } - Fields::Unnamed(unnamed) => { + Fields::Unnamed(paren, unnamed) => { + self.add_span(&paren.open(), &types::PARENTHESIS, &[])?; for field in unnamed { self.visit_unnamed_field(field)?; } + self.add_span(&paren.close(), &types::PARENTHESIS, &[])?; } Fields::Unit => {} } @@ -264,8 +292,13 @@ impl<'a> Visitor<'a> { fn visit_named_field(&mut self, item: &NamedField<'_>) -> Result<()> { self.visit_comment(&item.comment)?; self.add_span(&item.name, &types::PROPERTY, &[modifiers::DECLARATION])?; + self.add_span(&item.colon, &types::COLON, &[])?; self.visit_type(&item.ty)?; - self.visit_id(&item.id) + self.visit_id(&item.id)?; + if let Some(comma) = &item.comma { + self.add_span(comma, &types::COMMA, &[])?; + } + Ok(()) } fn visit_unnamed_field(&mut self, item: &UnnamedField<'_>) -> Result<()> { @@ -283,13 +316,17 @@ impl<'a> Visitor<'a> { fn visit_alias(&mut self, item: &TypeAlias<'_>) -> Result<()> { self.visit_comment(&item.comment)?; + self.add_span(&item.keyword, &types::KEYWORD, &[])?; self.add_span(&item.name, &types::TYPE_ALIAS, &[modifiers::DECLARATION])?; self.visit_generics(&item.generics)?; - self.visit_type(&item.target) + self.add_span(&item.equal, &types::EQUAL, &[])?; + self.visit_type(&item.target)?; + self.add_span(&item.semicolon, &types::SEMICOLON, &[]) } fn visit_const(&mut self, item: &Const<'_>) -> Result<()> { self.visit_comment(&item.comment)?; + self.add_span(&item.keyword, &types::KEYWORD, &[])?; self.add_span( &item.name, &types::VARIABLE, @@ -299,20 +336,86 @@ impl<'a> Visitor<'a> { modifiers::CONSTANT, ], )?; + self.add_span(&item.colon, &types::COLON, &[])?; self.visit_type(&item.ty)?; - self.visit_literal(&item.value) + self.add_span(&item.equal, &types::EQUAL, &[])?; + self.visit_literal(&item.value)?; + self.add_span(&item.semicolon, &types::SEMICOLON, &[]) } fn visit_type(&mut self, item: &Type<'_>) -> Result<()> { - self.add_span( - item, - &if matches!(item.value, DataType::External(_)) { - types::TYPE - } else { - types::BUILTIN_TYPE - }, - &[], - ) + match &item.value { + DataType::Bool + | DataType::U8 + | DataType::U16 + | DataType::U32 + | DataType::U64 + | DataType::U128 + | DataType::I8 + | DataType::I16 + | DataType::I32 + | DataType::I64 + | DataType::I128 + | DataType::F32 + | DataType::F64 + | DataType::String + | DataType::StringRef + | DataType::BoxString + | DataType::Bytes + | DataType::BytesRef + | DataType::BoxBytes => self.add_span(item, &types::BUILTIN_TYPE, &[]), + DataType::Vec(span, angle, ty) + | DataType::HashSet(span, angle, ty) + | DataType::Option(span, angle, ty) + | DataType::NonZero(span, angle, ty) => { + self.add_span(span, &types::BUILTIN_TYPE, &[])?; + self.add_span(&angle.open(), &types::ANGLE, &[])?; + self.visit_type(ty)?; + self.add_span(&angle.close(), &types::ANGLE, &[]) + } + DataType::HashMap(span, angle, comma, kv) => { + self.add_span(span, &types::BUILTIN_TYPE, &[])?; + self.add_span(&angle.open(), &types::ANGLE, &[])?; + self.visit_type(&kv.0)?; + self.add_span(comma, &types::COMMA, &[])?; + self.visit_type(&kv.1)?; + self.add_span(&angle.close(), &types::ANGLE, &[]) + } + DataType::Tuple(paren, types) => { + self.add_span(&paren.open(), &types::PARENTHESIS, &[])?; + for ty in types { + self.visit_type(ty)?; + } + self.add_span(&paren.close(), &types::PARENTHESIS, &[]) + } + DataType::Array(bracket, ty, semicolon, _) => { + self.add_span(&bracket.open(), &types::BRACKET, &[])?; + self.visit_type(ty)?; + self.add_span(semicolon, &types::SEMICOLON, &[])?; + self.add_span(&bracket.close(), &types::BRACKET, &[]) + } + DataType::External(ExternalType { + path, + name, + angle, + generics, + }) => { + for name in path { + self.add_span(name, &types::NAMESPACE, &[])?; + } + self.add_span(name, &types::TYPE, &[])?; + if let Some(angle) = angle { + self.add_span(&angle.open(), &types::ANGLE, &[])?; + } + for ty in generics { + self.visit_type(ty)?; + } + if let Some(angle) = angle { + self.add_span(&angle.close(), &types::ANGLE, &[])?; + } + Ok(()) + } + } } fn visit_literal(&mut self, item: &Literal) -> Result<()> { @@ -335,4 +438,8 @@ impl<'a> Visitor<'a> { Ok(()) } + + fn visit_import(&mut self, item: &Import<'_>) -> Result<()> { + self.add_span(&item.keyword, &types::KEYWORD, &[]) + } } diff --git a/crates/mabo-parser/src/lib.rs b/crates/mabo-parser/src/lib.rs index 2768540..223c8f8 100644 --- a/crates/mabo-parser/src/lib.rs +++ b/crates/mabo-parser/src/lib.rs @@ -17,7 +17,7 @@ //! ``` use std::{ - fmt::{self, Display}, + fmt::{self, Display, Write}, ops::Range, path::{Path, PathBuf}, }; @@ -27,13 +27,15 @@ pub use miette::{Diagnostic, LabeledSpan}; use miette::{IntoDiagnostic, NamedSource, Result}; use winnow::Parser; -use self::error::ParseSchemaError; +use self::{error::ParseSchemaError, token::Punctuation}; +use crate::token::Delimiter; pub mod error; mod ext; mod highlight; mod location; mod parser; +pub mod token; trait Print { /// Default indentation, 4 spaces. @@ -69,12 +71,27 @@ impl From> for Span { } } +impl From<&Range> for Span { + fn from(value: &Range) -> Self { + Self { + start: value.start, + end: value.end, + } + } +} + impl From for Range { fn from(value: Span) -> Self { value.start..value.end } } +impl Spanned for Span { + fn span(&self) -> Span { + *self + } +} + /// Implemented by any parsed schema element that can report the source code span that it originated /// from. This helps during error reporting to visualize the exact location of a problem. pub trait Spanned { @@ -233,8 +250,12 @@ impl<'a> Definition<'a> { pub struct Module<'a> { /// Optional module-level comment. pub comment: Comment<'a>, + /// The `mod` keyword to mark the module declaration. + pub keyword: token::Mod, /// Unique name of the module, within the current scope. pub name: Name<'a>, + /// Braces `{`...`}` around the contained definitions. + pub brace: token::Brace, /// List of definitions that are scoped within this module. pub definitions: Vec>, } @@ -243,14 +264,16 @@ impl Print for Module<'_> { fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { let Self { comment, + keyword, name, definitions, + .. } = self; comment.print(f, level)?; + keyword.print(f, level)?; - Self::indent(f, level)?; - writeln!(f, "mod {name} {{")?; + writeln!(f, " {name} {}", token::Brace::OPEN)?; for (i, definition) in definitions.iter().enumerate() { if i > 0 { @@ -260,7 +283,7 @@ impl Print for Module<'_> { } Self::indent(f, level)?; - f.write_str("}\n") + writeln!(f, "{}", token::Brace::CLOSE) } } @@ -290,6 +313,8 @@ pub struct Struct<'a> { pub comment: Comment<'a>, /// Optional attributes to customize the behavior. pub attributes: Attributes<'a>, + /// The `struct` keyword to mark the struct declaration. + pub keyword: token::Struct, /// Unique name for this struct (within its scope). pub name: Name<'a>, /// Potential generics. @@ -300,10 +325,10 @@ pub struct Struct<'a> { impl Print for Struct<'_> { fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { - let indent = Self::INDENT.repeat(level); let Self { comment, attributes, + keyword, name, generics, fields: kind, @@ -311,8 +336,11 @@ impl Print for Struct<'_> { comment.print(f, level)?; attributes.print(f, level)?; - write!(f, "{indent}struct {name}{generics}")?; + keyword.print(f, level)?; + + write!(f, " {name}{generics}")?; kind.print(f, level)?; + f.write_str("\n") } } @@ -344,10 +372,14 @@ pub struct Enum<'a> { pub comment: Comment<'a>, /// Optional attributes to customize the behavior. pub attributes: Attributes<'a>, + /// The `enum` keyword to mark the enum declaration. + pub keyword: token::Enum, /// Unique name for this enum, within its current scope. pub name: Name<'a>, /// Potential generics. pub generics: Generics<'a>, + /// Braces `{`...`}` around the variants. + pub brace: token::Brace, /// List of possible variants that the enum can represent. pub variants: Vec>, } @@ -357,16 +389,18 @@ impl Print for Enum<'_> { let Self { comment, attributes, + keyword, name, generics, variants, + .. } = self; comment.print(f, level)?; attributes.print(f, level)?; + keyword.print(f, level)?; - Self::indent(f, level)?; - writeln!(f, "enum {name}{generics} {{")?; + writeln!(f, " {name}{generics} {}", token::Brace::OPEN)?; for variant in variants { variant.print(f, level + 1)?; @@ -374,7 +408,7 @@ impl Print for Enum<'_> { } Self::indent(f, level)?; - f.write_str("}\n") + writeln!(f, "{}", token::Brace::CLOSE) } } @@ -395,6 +429,8 @@ pub struct Variant<'a> { pub fields: Fields<'a>, /// Identifier for this variant, that must be unique within the current enum. pub id: Option, + /// Trailing comma to separate variants (might be missing if this is the last element). + pub comma: Option, /// Source code location. span: Span, } @@ -406,7 +442,8 @@ impl Print for Variant<'_> { name, fields, id, - span: _, + comma, + .. } = self; comment.print(f, level)?; @@ -415,10 +452,12 @@ impl Print for Variant<'_> { f.write_str(name.get())?; fields.print(f, level)?; if let Some(id) = id { - write!(f, " {id},") - } else { - write!(f, ",") + write!(f, " {id}")?; + } + if let Some(comma) = comma { + write!(f, "{comma}")?; } + Ok(()) } } @@ -447,27 +486,36 @@ impl Display for Variant<'_> { pub struct TypeAlias<'a> { /// Optional element-level comment. pub comment: Comment<'a>, + /// The `type` keyword to mark the type alias declaration. + pub keyword: token::Type, /// Unique name of the type alias within the current scope. pub name: Name<'a>, /// Potential generic type arguments. pub generics: Generics<'a>, + /// Equal operator that assigns the target type. + pub equal: token::Equal, /// Original type that is being aliased. pub target: Type<'a>, + /// Trailing semicolon to complete the definition. + pub semicolon: token::Semicolon, } impl Print for TypeAlias<'_> { fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { let Self { comment, + keyword, name, generics, + equal, target, + semicolon, } = self; comment.print(f, level)?; + keyword.print(f, level)?; - Self::indent(f, level)?; - write!(f, "type {name}{generics} = {target};") + write!(f, " {name}{generics} {equal} {target}{semicolon}") } } @@ -489,13 +537,13 @@ pub enum Fields<'a> { /// c: i32 @3, /// } /// ``` - Named(Vec>), + Named(token::Brace, Vec>), /// List of types without an explicit name. /// /// ```txt /// Sample(u8 @1, bool @2, i32 @3) /// ``` - Unnamed(Vec>), + Unnamed(token::Parenthesis, Vec>), /// No attached value. /// /// ```txt @@ -507,18 +555,18 @@ pub enum Fields<'a> { impl Print for Fields<'_> { fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { match self { - Fields::Named(fields) => { - f.write_str(" {\n")?; + Fields::Named(_, fields) => { + writeln!(f, " {}", token::Brace::OPEN)?; for field in fields { field.print(f, level + 1)?; - f.write_str(",\n")?; + f.write_char('\n')?; } Self::indent(f, level)?; - f.write_str("}") + f.write_char(token::Brace::CLOSE) } - Fields::Unnamed(elements) => concat(f, "(", elements, ", ", ")"), + Fields::Unnamed(_, elements) => concat::(f, elements, " "), Fields::Unit => Ok(()), } } @@ -545,10 +593,14 @@ pub struct NamedField<'a> { pub comment: Comment<'a>, /// Unique name for this field, within the current element. pub name: Name<'a>, + /// Colon to separate the field name from the type. + pub colon: token::Colon, /// Data type that defines the shape of the contained data. pub ty: Type<'a>, /// Identifier for this field, that must be unique within the current element. pub id: Option, + /// Trailing comma to separate fields (might be missing if this is the last element). + pub comma: Option, /// Source code location. span: Span, } @@ -558,20 +610,27 @@ impl Print for NamedField<'_> { let Self { comment, name, + colon, ty, id, - span: _, + comma, + .. } = self; comment.print(f, level)?; Self::indent(f, level)?; + write!(f, "{name}{colon} {ty}")?; if let Some(id) = id { - write!(f, "{name}: {ty} {id}") - } else { - write!(f, "{name}: {ty}") + write!(f, " {id}")?; } + + if let Some(comma) = comma { + comma.fmt(f)?; + } + + Ok(()) } } @@ -601,6 +660,8 @@ pub struct UnnamedField<'a> { pub ty: Type<'a>, /// Identifier for this field, that must be unique within the current element. pub id: Option, + /// Trailing comma to separate fields (might be missing if this is the last element). + pub comma: Option, /// Source code location. span: Span, } @@ -613,12 +674,18 @@ impl Spanned for UnnamedField<'_> { impl Display for UnnamedField<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { ty, id, span: _ } = self; + let Self { ty, id, comma, .. } = self; + write!(f, "{ty}")?; + if let Some(id) = id { - write!(f, "{ty} {id}") - } else { - write!(f, "{ty}") + write!(f, " {id}")?; + } + + if let Some(comma) = comma { + comma.fmt(f)?; } + + Ok(()) } } @@ -708,7 +775,9 @@ impl Print for Attributes<'_> { let values = &self.0; Self::indent(f, level)?; - concat(f, "#[", values, ", ", "]\n") + f.write_str(token::Pound::VALUE)?; + concat::(f, values, ", ")?; + f.write_char('\n') } } @@ -758,7 +827,7 @@ impl Print for AttributeValue<'_> { match self { Self::Unit => Ok(()), Self::Single(lit) => write!(f, " = {lit}"), - Self::Multi(attrs) => concat(f, "(", attrs, ", ", ")"), + Self::Multi(attrs) => concat::(f, attrs, ", "), } } } @@ -838,25 +907,25 @@ pub enum DataType<'a> { /// Reference version (slice) of `u8` bytes. BytesRef, /// Vector of another data type. - Vec(Box>), + Vec(Span, token::Angle, Box>), /// Key-value hash map of data types. - HashMap(Box<(Type<'a>, Type<'a>)>), + HashMap(Span, token::Angle, token::Comma, Box<(Type<'a>, Type<'a>)>), /// Hash set of data types (each entry is unique). - HashSet(Box>), + HashSet(Span, token::Angle, Box>), /// Optional value. - Option(Box>), + Option(Span, token::Angle, Box>), /// Non-zero value. /// - Integers: `n > 0` /// - Collections: `len() > 0` - NonZero(Box>), + NonZero(Span, token::Angle, Box>), /// Boxed version of a string that is immutable. BoxString, /// Boxed version of a byte vector that is immutable. BoxBytes, /// Fixed size list of up to 12 types. - Tuple(Vec>), + Tuple(token::Parenthesis, Vec>), /// Continuous list of values with a single time and known length. - Array(Box>, u32), + Array(token::Bracket, Box>, token::Semicolon, u32), /// Any external, non-standard data type (like a user defined struct or enum). External(ExternalType<'a>), } @@ -881,15 +950,15 @@ impl Display for DataType<'_> { Self::StringRef => f.write_str("&string"), Self::Bytes => f.write_str("bytes"), Self::BytesRef => f.write_str("&bytes"), - Self::Vec(t) => write!(f, "vec<{t}>"), - Self::HashMap(kv) => write!(f, "hash_map<{}, {}>", kv.0, kv.1), - Self::HashSet(t) => write!(f, "hash_set<{t}>"), - Self::Option(t) => write!(f, "option<{t}>"), - Self::NonZero(t) => write!(f, "non_zero<{t}>"), + Self::Vec(_, _, t) => write!(f, "vec<{t}>"), + Self::HashMap(_, _, _, kv) => write!(f, "hash_map<{}, {}>", kv.0, kv.1), + Self::HashSet(_, _, t) => write!(f, "hash_set<{t}>"), + Self::Option(_, _, t) => write!(f, "option<{t}>"), + Self::NonZero(_, _, t) => write!(f, "non_zero<{t}>"), Self::BoxString => f.write_str("box"), Self::BoxBytes => f.write_str("box"), - Self::Tuple(l) => concat(f, "(", l, ", ", ")"), - Self::Array(t, size) => write!(f, "[{t}; {size}]"), + Self::Tuple(_, l) => concat::(f, l, ", "), + Self::Array(_, t, _, size) => write!(f, "[{t}; {size}]"), Self::External(t) => t.fmt(f), } } @@ -905,6 +974,8 @@ pub struct ExternalType<'a> { pub path: Vec>, /// Unique name of the type within the current scope (or the module if prefixed with a path). pub name: Name<'a>, + /// Angles `<`...`>` to delimit the generic type parameters. + pub angle: Option, /// Potential generic type arguments. pub generics: Vec>, } @@ -915,13 +986,14 @@ impl Display for ExternalType<'_> { path, name, generics, + .. } = self; for segment in path { write!(f, "{segment}::")?; } name.fmt(f)?; - concat(f, "<", generics, ", ", ">") + concat::(f, generics, ", ") } } @@ -935,7 +1007,7 @@ pub struct Generics<'a>(pub Vec>); impl Display for Generics<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - concat(f, "<", &self.0, ", ", ">") + concat::(f, &self.0, ", ") } } @@ -1032,27 +1104,39 @@ impl AsRef for Name<'_> { pub struct Const<'a> { /// Optional element-level comment. pub comment: Comment<'a>, + /// The `const` keyword to mark the constant declaration. + pub keyword: token::Const, /// Unique identifier of this constant. pub name: Name<'a>, + /// Colon to separate the const name from the type. + pub colon: token::Colon, /// Type of the value. pub ty: Type<'a>, + /// Equal operator that assigns the literal value. + pub equal: token::Equal, /// Literal value that this declaration represents. pub value: Literal, + /// Trailing semicolon to complete the definition. + pub semicolon: token::Semicolon, } impl Print for Const<'_> { fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { let Self { comment, + keyword, name, + colon, ty, + equal, value, + semicolon, } = self; comment.print(f, level)?; + keyword.print(f, level)?; - Self::indent(f, level)?; - write!(f, "const {name}: {ty} = {value};") + write!(f, " {name}{colon} {ty} {equal} {value}{semicolon}") } } @@ -1123,6 +1207,8 @@ impl Display for LiteralValue { /// Import declaration for an external schema. #[derive(Debug, PartialEq)] pub struct Import<'a> { + /// The `use` keyword to mark the import declaration. + pub keyword: token::Use, /// Full import path as it was found in the original schema file. pub full: Name<'a>, /// Individual elements that form the import path. @@ -1130,16 +1216,22 @@ pub struct Import<'a> { /// Optional final element that allows to fully import the type, making it look as it would be /// defined in the current schema. pub element: Option>, + /// Trailing semicolon to complete the definition. + pub semicolon: token::Semicolon, } impl Print for Import<'_> { fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { let Self { - segments, element, .. + keyword, + segments, + element, + semicolon, + .. } = self; - Self::indent(f, level)?; - f.write_str("use ")?; + keyword.print(f, level)?; + f.write_str(" ")?; for (i, segment) in segments.iter().enumerate() { if i > 0 { @@ -1152,22 +1244,20 @@ impl Print for Import<'_> { write!(f, "::{element}")?; } - f.write_str(";") + write!(f, "{semicolon}") } } -fn concat( +fn concat( f: &mut fmt::Formatter<'_>, - open: &str, values: &[impl Display], sep: &str, - close: &str, ) -> fmt::Result { if values.is_empty() { return Ok(()); } - f.write_str(open)?; + f.write_char(D::OPEN)?; for (i, value) in values.iter().enumerate() { if i > 0 { @@ -1176,5 +1266,5 @@ fn concat( value.fmt(f)?; } - f.write_str(close) + f.write_char(D::CLOSE) } diff --git a/crates/mabo-parser/src/parser/aliases.rs b/crates/mabo-parser/src/parser/aliases.rs index ea59266..eb34c0a 100644 --- a/crates/mabo-parser/src/parser/aliases.rs +++ b/crates/mabo-parser/src/parser/aliases.rs @@ -3,7 +3,7 @@ use std::ops::Range; use mabo_derive::{ParserError, ParserErrorCause}; use winnow::{ ascii::{alphanumeric0, space0, space1}, - combinator::{cut_err, delimited, opt, preceded}, + combinator::{cut_err, opt, preceded, terminated}, error::ErrorKind, stream::Location, token::one_of, @@ -11,7 +11,11 @@ use winnow::{ }; use super::{generics, types, Input, ParserExt, Result}; -use crate::{highlight, Comment, Name, TypeAlias}; +use crate::{ + highlight, + token::{self, Punctuation}, + Comment, Name, TypeAlias, +}; /// Encountered an invalid `type` alias declaration. #[derive(Debug, ParserError)] @@ -63,31 +67,34 @@ pub enum Cause { } pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> { - preceded( - ("type", space1), + ( + terminated(token::Type::NAME.span(), space1), cut_err(( parse_name, opt(generics::parse.map_err(Cause::Generics)).map(Option::unwrap_or_default), - delimited( - (space0, '='), - preceded(space0, types::parse.map_err(Cause::from)), - (space0, ';'), - ), + preceded(space0, token::Equal::VALUE.span()), + preceded(space0, types::parse.map_err(Cause::from)), + preceded(space0, token::Semicolon::VALUE.span()), )), ) - .parse_next(input) - .map(|(name, generics, target)| TypeAlias { - comment: Comment::default(), - name, - generics, - target, - }) - .map_err(|e| { - e.map(|cause| ParseError { - at: input.location()..input.location(), - cause, + .parse_next(input) + .map( + |(keyword, (name, generics, equal, target, semicolon))| TypeAlias { + comment: Comment::default(), + keyword: keyword.into(), + name, + generics, + equal: equal.into(), + target, + semicolon: semicolon.into(), + }, + ) + .map_err(|e| { + e.map(|cause| ParseError { + at: input.location()..input.location(), + cause, + }) }) - }) } fn parse_name<'i>(input: &mut Input<'i>) -> Result, Cause> { diff --git a/crates/mabo-parser/src/parser/consts.rs b/crates/mabo-parser/src/parser/consts.rs index fe7d7bb..3aac8b7 100644 --- a/crates/mabo-parser/src/parser/consts.rs +++ b/crates/mabo-parser/src/parser/consts.rs @@ -3,7 +3,7 @@ use std::ops::Range; use mabo_derive::{ParserError, ParserErrorCause}; use winnow::{ ascii::{space0, space1}, - combinator::{cut_err, delimited, preceded, terminated}, + combinator::{cut_err, preceded, terminated}, error::ErrorKind, stream::{Location, Stream}, token::{one_of, take_while}, @@ -11,7 +11,11 @@ use winnow::{ }; use super::{literals, types, Input, ParserExt, Result}; -use crate::{highlight, location, Comment, Const, Name}; +use crate::{ + highlight, location, + token::{self, Punctuation}, + Comment, Const, Name, +}; /// Encountered an invalid `const` declaration. #[derive(Debug, ParserError)] @@ -78,31 +82,38 @@ pub enum Cause { pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> { let start = input.checkpoint(); - preceded( - ("const", space1), + ( + terminated(token::Const::NAME.span(), space1), cut_err(( - terminated(parse_name, (':', space0)), - types::parse.map_err(Cause::from), - delimited( - (space0, '=', space0), - literals::parse.map_err(Cause::from), - ';'.map_err_loc(|at, ()| Cause::UnexpectedChar { at, expected: ';' }), - ), + parse_name, + token::Colon::VALUE.span(), + preceded(space0, types::parse.map_err(Cause::from)), + preceded(space0, token::Equal::VALUE.span()), + preceded(space0, literals::parse.map_err(Cause::from)), + token::Semicolon::VALUE + .span() + .map_err_loc(|at, ()| Cause::UnexpectedChar { at, expected: ';' }), )), ) - .parse_next(input) - .map(|(name, ty, value)| Const { - comment: Comment::default(), - name, - ty, - value, - }) - .map_err(|e| { - e.map(|cause| ParseError { - at: location::from_until(*input, &start, [';']), - cause, + .parse_next(input) + .map( + |(keyword, (name, colon, ty, equal, value, semicolon))| Const { + comment: Comment::default(), + keyword: keyword.into(), + name, + colon: colon.into(), + ty, + equal: equal.into(), + value, + semicolon: semicolon.into(), + }, + ) + .map_err(|e| { + e.map(|cause| ParseError { + at: location::from_until(*input, &start, [';']), + cause, + }) }) - }) } pub(super) fn parse_name<'i>(input: &mut Input<'i>) -> Result, Cause> { diff --git a/crates/mabo-parser/src/parser/enums.rs b/crates/mabo-parser/src/parser/enums.rs index abf0907..ce19df3 100644 --- a/crates/mabo-parser/src/parser/enums.rs +++ b/crates/mabo-parser/src/parser/enums.rs @@ -3,7 +3,7 @@ use std::ops::Range; use mabo_derive::{ParserError, ParserErrorCause}; use winnow::{ ascii::{alphanumeric0, space0, space1}, - combinator::{cut_err, opt, preceded, separated, terminated}, + combinator::{cut_err, opt, preceded, repeat, terminated}, error::ErrorKind, stream::Location, token::one_of, @@ -11,7 +11,11 @@ use winnow::{ }; use super::{comments, fields, generics, ids, ws, Input, ParserExt, Result}; -use crate::{highlight, Attributes, Comment, Enum, Name, Variant}; +use crate::{ + highlight, + token::{self, Delimiter, Punctuation}, + Attributes, Comment, Enum, Name, Variant, +}; /// Encountered an invalid `enum` declaration. #[derive(Debug, ParserError)] @@ -85,28 +89,30 @@ pub enum Cause { } pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> { - preceded( - ("enum", space1), + ( + terminated(token::Enum::NAME.span(), space1), cut_err(( parse_name, opt(generics::parse.map_err(Cause::from)).map(Option::unwrap_or_default), preceded(space0, parse_variants), )), ) - .parse_next(input) - .map(|(name, generics, variants)| Enum { - comment: Comment::default(), - attributes: Attributes::default(), - name, - generics, - variants, - }) - .map_err(|e| { - e.map(|cause| ParseError { - at: input.location()..input.location(), - cause, + .parse_next(input) + .map(|(keyword, (name, generics, (brace, variants)))| Enum { + comment: Comment::default(), + attributes: Attributes::default(), + keyword: keyword.into(), + name, + generics, + brace, + variants, + }) + .map_err(|e| { + e.map(|cause| ParseError { + at: input.location()..input.location(), + cause, + }) }) - }) } pub(super) fn parse_name<'i>(input: &mut Input<'i>) -> Result, Cause> { @@ -122,15 +128,13 @@ pub(super) fn parse_name<'i>(input: &mut Input<'i>) -> Result, Cause> { }) } -fn parse_variants<'i>(input: &mut Input<'i>) -> Result>, Cause> { - preceded( - '{', - cut_err(terminated( - terminated(separated(1.., parse_variant, ws(',')), opt(ws(','))), - ws('}'), - )), +fn parse_variants<'i>(input: &mut Input<'i>) -> Result<(token::Brace, Vec>), Cause> { + ( + token::Brace::OPEN.span(), + cut_err((repeat(1.., parse_variant), ws(token::Brace::CLOSE.span()))), ) - .parse_next(input) + .parse_next(input) + .map(|(brace_open, (variants, brace_close))| ((brace_open, brace_close).into(), variants)) } fn parse_variant<'i>(input: &mut Input<'i>) -> Result, Cause> { @@ -140,15 +144,17 @@ fn parse_variant<'i>(input: &mut Input<'i>) -> Result, Cause> { preceded(space0, parse_variant_name.with_span()), preceded(space0, fields::parse.map_err(Cause::from)), opt(preceded(space0, ids::parse.map_err(Cause::from))), + opt(ws(token::Comma::VALUE.span())), ) .with_span(), ) .parse_next(input) - .map(|(comment, ((name, fields, id), span))| Variant { + .map(|(comment, ((name, fields, id, comma), span))| Variant { comment, name: name.into(), fields, id, + comma: comma.map(Into::into), span: span.into(), }) } diff --git a/crates/mabo-parser/src/parser/fields.rs b/crates/mabo-parser/src/parser/fields.rs index a758dbd..869003b 100644 --- a/crates/mabo-parser/src/parser/fields.rs +++ b/crates/mabo-parser/src/parser/fields.rs @@ -3,7 +3,7 @@ use std::ops::Range; use mabo_derive::{ParserError, ParserErrorCause}; use winnow::{ ascii::space0, - combinator::{cut_err, delimited, opt, peek, preceded, separated, terminated}, + combinator::{cut_err, opt, peek, preceded, repeat}, dispatch, error::ErrorKind, stream::{Location, Stream}, @@ -12,7 +12,11 @@ use winnow::{ }; use super::{comments, ids, types, ws, Input, ParserExt, Result}; -use crate::{highlight, location, Fields, Name, NamedField, UnnamedField}; +use crate::{ + highlight, location, + token::{self, Delimiter, Punctuation}, + Fields, Name, NamedField, UnnamedField, +}; /// Encountered an invalid field declaration. #[derive(Debug, ParserError)] @@ -73,8 +77,8 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> dispatch!( peek(any); - '{' => parse_named.map(Fields::Named), - '(' => parse_unnamed.map(Fields::Unnamed), + '{' => parse_named.map(|(brace, fields)| Fields::Named(brace, fields)), + '(' => parse_unnamed.map(|(paren, fields)| Fields::Unnamed(paren, fields)), _ => parse_unit.map(|()| Fields::Unit), ) .parse_next(input) @@ -86,26 +90,30 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> }) } -fn parse_named<'i>(input: &mut Input<'i>) -> Result>, Cause> { - preceded( - '{', - cut_err(terminated( - terminated(separated(1.., parse_named_field, ws(',')), opt(ws(','))), - ws('}'), +fn parse_named<'i>(input: &mut Input<'i>) -> Result<(token::Brace, Vec>), Cause> { + ( + token::Brace::OPEN.span(), + cut_err(( + repeat(1.., parse_named_field), + ws(token::Brace::CLOSE.span()), )), ) - .parse_next(input) + .parse_next(input) + .map(|(brace_open, (fields, brace_close))| ((brace_open, brace_close).into(), fields)) } -fn parse_unnamed<'i>(input: &mut Input<'i>) -> Result>, Cause> { - preceded( - '(', - cut_err(terminated( - terminated(separated(1.., parse_unnamed_field, ws(',')), opt(ws(','))), - ws(')'), +fn parse_unnamed<'i>( + input: &mut Input<'i>, +) -> Result<(token::Parenthesis, Vec>), Cause> { + ( + token::Parenthesis::OPEN.span(), + cut_err(( + repeat(1.., parse_unnamed_field), + ws(token::Parenthesis::CLOSE.span()), )), ) - .parse_next(input) + .parse_next(input) + .map(|(paren_open, (fields, paren_close))| ((paren_open, paren_close).into(), fields)) } fn parse_unit(input: &mut Input<'_>) -> Result<(), Cause> { @@ -116,12 +124,14 @@ fn parse_unnamed_field<'i>(input: &mut Input<'i>) -> Result, Ca ( ws(types::parse.map_err(Cause::from)), opt(preceded(space0, ids::parse.map_err(Cause::from))), + opt(ws(token::Comma::VALUE.span())), ) .with_span() .parse_next(input) - .map(|((ty, id), span)| UnnamedField { + .map(|((ty, id, comma), span)| UnnamedField { ty, id, + comma: comma.map(Into::into), span: span.into(), }) } @@ -130,20 +140,26 @@ fn parse_named_field<'i>(input: &mut Input<'i>) -> Result, Cause> ( ws(comments::parse.map_err(Cause::from)), ( - delimited(space0, parse_field_name, ':'), + preceded(space0, parse_field_name), + preceded(space0, token::Colon::VALUE.span()), preceded(space0, types::parse.map_err(Cause::from)), opt(preceded(space0, ids::parse.map_err(Cause::from))), + opt(ws(token::Comma::VALUE.span())), ) .with_span(), ) .parse_next(input) - .map(|(comment, ((name, ty, id), span))| NamedField { - comment, - name, - ty, - id, - span: span.into(), - }) + .map( + |(comment, ((name, colon, ty, id, comma), span))| NamedField { + comment, + name, + colon: colon.into(), + ty, + id, + comma: comma.map(Into::into), + span: span.into(), + }, + ) } fn parse_field_name<'i>(input: &mut Input<'i>) -> Result, Cause> { diff --git a/crates/mabo-parser/src/parser/imports.rs b/crates/mabo-parser/src/parser/imports.rs index 121557f..4b7370c 100644 --- a/crates/mabo-parser/src/parser/imports.rs +++ b/crates/mabo-parser/src/parser/imports.rs @@ -11,7 +11,11 @@ use winnow::{ }; use super::{enums, structs, Input, ParserExt, Result}; -use crate::{highlight, location, Import, Name}; +use crate::{ + highlight, location, + token::{self, Punctuation}, + Import, Name, +}; /// Encountered an invalid `use` declaration. #[derive(Debug, ParserError)] @@ -65,13 +69,13 @@ pub enum Cause { pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> { let start = input.checkpoint(); - preceded( - ("use", space1), - cut_err(terminated( + ( + terminated(token::Use::NAME.span(), space1), + cut_err(( ( - separated(1.., parse_segment, "::"), + separated(1.., parse_segment, token::DoubleColon::VALUE.span()), opt(preceded( - "::", + token::DoubleColon::VALUE.span(), alt(( structs::parse_name.map_err(Cause::from), enums::parse_name.map_err(Cause::from), @@ -80,21 +84,25 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> ) .with_recognized() .with_span(), - ';', + token::Semicolon::VALUE.span(), )), ) - .parse_next(input) - .map(|(((segments, element), full), range)| Import { - full: (full, range).into(), - segments, - element, - }) - .map_err(|e| { - e.map(|cause| ParseError { - at: location::from_until(*input, &start, [';']), - cause, + .parse_next(input) + .map( + |(keyword, ((((segments, element), full), range), semicolon))| Import { + keyword: keyword.into(), + full: (full, range).into(), + segments, + element, + semicolon: semicolon.into(), + }, + ) + .map_err(|e| { + e.map(|cause| ParseError { + at: location::from_until(*input, &start, [';']), + cause, + }) }) - }) } pub(super) fn parse_segment<'i>(input: &mut Input<'i>) -> Result, Cause> { diff --git a/crates/mabo-parser/src/parser/modules.rs b/crates/mabo-parser/src/parser/modules.rs index 25af73f..3c01576 100644 --- a/crates/mabo-parser/src/parser/modules.rs +++ b/crates/mabo-parser/src/parser/modules.rs @@ -11,7 +11,12 @@ use winnow::{ }; use super::{parse_definition, ws, Input, ParserExt, Result}; -use crate::{error::ParseDefinitionError, highlight, location, Comment, Module, Name}; +use crate::{ + error::ParseDefinitionError, + highlight, location, + token::{self, Delimiter}, + Comment, Module, Name, +}; /// Encountered an invalid `mod` declaration. #[derive(Debug, ParserError)] @@ -62,31 +67,31 @@ pub enum Cause { pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> { let start = input.checkpoint(); - preceded( - ("mod", space1), + ( + terminated(token::Mod::NAME.span(), space1), cut_err(( parse_name, - preceded( - (space0, '{'), - terminated( - repeat(0.., ws(parse_definition.map_err(Cause::from))), - ws('}'), - ), - ), + preceded(space0, token::Brace::OPEN.span()), + repeat(0.., ws(parse_definition.map_err(Cause::from))), + ws(token::Brace::CLOSE.span()), )), ) - .parse_next(input) - .map(|(name, definitions)| Module { - comment: Comment::default(), - name, - definitions, - }) - .map_err(|e| { - e.map(|cause| ParseError { - at: location::from_until(*input, &start, ['}', '\n']), - cause, + .parse_next(input) + .map( + |(keyword, (name, brace_open, definitions, brace_close))| Module { + comment: Comment::default(), + keyword: keyword.into(), + name, + brace: (brace_open, brace_close).into(), + definitions, + }, + ) + .map_err(|e| { + e.map(|cause| ParseError { + at: location::from_until(*input, &start, ['}', '\n']), + cause, + }) }) - }) } fn parse_name<'i>(input: &mut Input<'i>) -> Result, Cause> { diff --git a/crates/mabo-parser/src/parser/structs.rs b/crates/mabo-parser/src/parser/structs.rs index ad815dd..5ad1ba9 100644 --- a/crates/mabo-parser/src/parser/structs.rs +++ b/crates/mabo-parser/src/parser/structs.rs @@ -3,7 +3,7 @@ use std::ops::Range; use mabo_derive::{ParserError, ParserErrorCause}; use winnow::{ ascii::{alphanumeric0, space0, space1}, - combinator::{cut_err, opt, preceded}, + combinator::{cut_err, opt, preceded, terminated}, error::ErrorKind, stream::{Location, Stream}, token::one_of, @@ -11,7 +11,7 @@ use winnow::{ }; use super::{fields, generics, Input, ParserExt, Result}; -use crate::{highlight, location, Attributes, Comment, Name, Struct}; +use crate::{highlight, location, token, Attributes, Comment, Name, Struct}; /// Encountered an invalid `struct` declaration. #[derive(Debug, ParserError)] @@ -65,28 +65,29 @@ pub enum Cause { pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result, ParseError> { let start = input.checkpoint(); - preceded( - ("struct", space1), + ( + terminated(token::Struct::NAME.span(), space1), cut_err(( parse_name, opt(generics::parse.map_err(Cause::Generics)).map(Option::unwrap_or_default), preceded(space0, fields::parse.map_err(Cause::Fields)), )), ) - .parse_next(input) - .map(|(name, generics, kind)| Struct { - comment: Comment::default(), - attributes: Attributes::default(), - name, - generics, - fields: kind, - }) - .map_err(|e| { - e.map(|cause| ParseError { - at: location::from_until(*input, &start, ['}', '\n']), - cause, + .parse_next(input) + .map(|(keyword, (name, generics, kind))| Struct { + comment: Comment::default(), + attributes: Attributes::default(), + keyword: keyword.into(), + name, + generics, + fields: kind, + }) + .map_err(|e| { + e.map(|cause| ParseError { + at: location::from_until(*input, &start, ['}', '\n']), + cause, + }) }) - }) } pub(super) fn parse_name<'i>(input: &mut Input<'i>) -> Result, Cause> { diff --git a/crates/mabo-parser/src/parser/types.rs b/crates/mabo-parser/src/parser/types.rs index 0c98e7c..eb6a368 100644 --- a/crates/mabo-parser/src/parser/types.rs +++ b/crates/mabo-parser/src/parser/types.rs @@ -3,7 +3,7 @@ use std::ops::Range; use mabo_derive::{ParserError, ParserErrorCause}; use winnow::{ ascii::{dec_uint, space0}, - combinator::{alt, cut_err, empty, fail, opt, preceded, separated, separated_pair, terminated}, + combinator::{alt, cut_err, empty, fail, opt, preceded, separated, terminated}, dispatch, error::ErrorKind, stream::Location, @@ -12,7 +12,11 @@ use winnow::{ }; use super::{imports, ws, Input, ParserExt, Result}; -use crate::{highlight, DataType, ExternalType, Name, Type}; +use crate::{ + highlight, + token::{self, Delimiter, Punctuation}, + DataType, ExternalType, Name, Type, +}; /// Encountered an invalid type definition. #[derive(Debug, ParserError)] @@ -98,76 +102,116 @@ fn parse_basic<'i>(input: &mut Input<'i>) -> Result, Cause> { } fn parse_generic<'i>(input: &mut Input<'i>) -> Result, Cause> { - terminated( - dispatch! { - terminated(take_while(3.., ('a'..='z', '_')), '<'); - "vec" => cut_err(parse.map_err(Cause::from)) - .map(|t| DataType::Vec(Box::new(t))), - "hash_map" => cut_err(separated_pair( - parse.map_err(Cause::from), - (',', space0), - parse.map_err(Cause::from), - )) - .map(|kv| DataType::HashMap(Box::new(kv))), - "hash_set" => cut_err(parse.map_err(Cause::from)) - .map(|t| DataType::HashSet(Box::new(t))), - "option" => cut_err(parse.map_err(Cause::from)) - .map(|t| DataType::Option(Box::new(t))), - "non_zero" => cut_err(parse.map_err(Cause::from)) - .map(|t| DataType::NonZero(Box::new(t))), - _ => fail, - }, - '>', - ) + dispatch! { + (take_while(3.., ('a'..='z', '_')).with_span(), token::Angle::OPEN.span()); + (("vec", ref span), ref open) => cut_err(( + parse.map_err(Cause::from), + token::Angle::CLOSE.span(), + )) + .map(|(ty, close)| DataType::Vec(span.into(), (open, &close).into(), Box::new(ty))), + (("hash_map", ref span), ref open) => cut_err(( + parse.map_err(Cause::from), + preceded(space0, token::Comma::VALUE.span()), + preceded(space0, parse.map_err(Cause::from)), + token::Angle::CLOSE.span(), + )) + .map(|(key, comma, value, close)| DataType::HashMap( + span.into(), + (open, &close).into(), + comma.into(), + Box::new((key, value)), + )), + (("hash_set", ref span), ref open) => cut_err(( + parse.map_err(Cause::from), + token::Angle::CLOSE.span(), + )) + .map(|(ty, close)| DataType::HashSet(span.into(), (open, &close).into(), Box::new(ty))), + (("option", ref span), ref open) => cut_err(( + parse.map_err(Cause::from), + token::Angle::CLOSE.span(), + )) + .map(|(ty, close)| DataType::Option(span.into(), (open, &close).into(), Box::new(ty))), + (("non_zero", ref span), ref open) => cut_err(( + parse.map_err(Cause::from), + token::Angle::CLOSE.span(), + )) + .map(|(ty, close)| DataType::NonZero(span.into(), (open, &close).into(), Box::new(ty))), + _ => fail, + } .parse_next(input) } fn parse_tuple<'i>(input: &mut Input<'i>) -> Result, Cause> { - preceded( - '(', - cut_err(terminated( - separated(0.., ws(parse.map_err(Cause::from)), ws(',')), - ws(')'), + ( + token::Parenthesis::OPEN.span(), + cut_err(( + separated( + 0.., + ws(parse.map_err(Cause::from)), + ws(token::Comma::VALUE.span()), + ), + ws(token::Parenthesis::CLOSE.span()), )), ) - .parse_next(input) - .map(DataType::Tuple) + .parse_next(input) + .map(|(paren_open, (ty, paren_close))| { + DataType::Tuple((paren_open, paren_close).into(), ty) + }) } fn parse_array<'i>(input: &mut Input<'i>) -> Result, Cause> { - preceded( - '[', - cut_err(terminated( - separated_pair(ws(parse.map_err(Cause::from)), ws(';'), ws(dec_uint)), - ws(']'), + ( + token::Bracket::OPEN.span(), + cut_err(( + ws(parse.map_err(Cause::from)), + ws(token::Semicolon::VALUE.span()), + ws(dec_uint), + ws(token::Bracket::CLOSE.span()), )), ) - .parse_next(input) - .map(|(t, size)| DataType::Array(Box::new(t), size)) + .parse_next(input) + .map(|(bracket_open, (ty, semicolon, size, bracket_close))| { + DataType::Array( + (bracket_open, bracket_close).into(), + Box::new(ty), + semicolon.into(), + size, + ) + }) } fn parse_external<'i>(input: &mut Input<'i>) -> Result, Cause> { ( opt(terminated( - separated(1.., imports::parse_segment.map_err(Cause::from), "::"), - "::", + separated( + 1.., + imports::parse_segment.map_err(Cause::from), + token::DoubleColon::VALUE, + ), + token::DoubleColon::VALUE, )) .map(Option::unwrap_or_default), parse_external_name, - opt(preceded( - '<', - cut_err(terminated( - separated(1.., ws(parse.map_err(Cause::from)), ws(',')), - ws('>'), + opt(( + token::Angle::OPEN.span(), + cut_err(( + separated(1.., ws(parse.map_err(Cause::from)), ws(token::Comma::VALUE)), + ws(token::Angle::CLOSE.span()), )), - )) - .map(Option::unwrap_or_default), + )), ) .parse_next(input) - .map(|(path, name, generics)| ExternalType { - path, - name, - generics, + .map(|(path, name, generics)| { + let (angle, generics) = match generics { + Some((open, (generics, close))) => (Some((open, close).into()), generics), + None => (None, Vec::default()), + }; + ExternalType { + path, + name, + angle, + generics, + } }) } diff --git a/crates/mabo-parser/src/token.rs b/crates/mabo-parser/src/token.rs new file mode 100644 index 0000000..063782b --- /dev/null +++ b/crates/mabo-parser/src/token.rs @@ -0,0 +1,234 @@ +//! Tokens of the Mabo schema language that represent keywords, punctuation and delimiters. + +use std::{ + fmt::{self, Display}, + ops::Range, +}; + +use mabo_derive::Debug; + +use crate::{Print, Span, Spanned}; + +macro_rules! define_keywords { + ($(#[$doc:meta] $name:ident $token:literal)*) => { + $( + #[$doc] + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub struct $name { + span: Span, + } + + impl Spanned for $name { + fn span(&self) -> Span { + self.span + } + } + + impl Print for $name { + fn print(&self, f: &mut fmt::Formatter<'_>, level: usize) -> fmt::Result { + Self::indent(f, level)?; + f.write_str($token) + } + } + + impl Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.print(f, 0) + } + } + + impl<'a> From> for $name { + fn from(span: Range) -> Self { + Self { + span: span.into(), + } + } + } + + impl $name { + /// Literal raw name of this keyword. + pub const NAME: &'static str = $token; + } + )* + }; +} + +/// A textual element that is used to separate repeated elements in the schema, or act as a form of +/// prefix. +pub trait Punctuation { + /// String that makes up the punctuation element. + const VALUE: &'static str; +} + +macro_rules! define_punctuation { + ($(#[$doc:meta] $name:ident $token:literal)*) => { + $( + #[$doc] + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub struct $name { + span: Span, + } + + impl Spanned for $name { + fn span(&self) -> Span { + self.span + } + } + + impl<'a> From> for $name { + fn from(span: Range) -> Self { + Self { + span: span.into(), + } + } + } + + impl Default for $name { + fn default() -> Self { + Self { + span: Span { + start: 0, + end: 0, + } + } + } + } + + impl Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str($token) + } + } + + impl Punctuation for $name { + /// Literal raw string of this punctuation. + const VALUE: &'static str = $token; + } + )* + }; +} + +/// Delimiters surround other elements with an opening and closing character. +pub trait Delimiter { + /// Opening character of the delimiter. + const OPEN: char; + /// Closing character of the delimiter. + const CLOSE: char; + + /// Get the location of the opening token. + fn open(&self) -> Span; + /// Get the location of the closing token. + fn close(&self) -> Span; + /// Get a combined span that goes from the open to the close token. + fn range(&self) -> Span; +} + +macro_rules! define_delimiters { + ($(#[$doc:meta] $name:ident $token_open:literal $token_close:literal)*) => { + $( + #[$doc] + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub struct $name { + open: Span, + close: Span, + } + + impl<'a> From<(Range, Range)> for $name { + fn from((open, close): (Range, Range)) -> Self { + Self { + open: open.into(), + close: close.into(), + } + } + } + + impl<'a> From<(&Range, &Range)> for $name { + fn from((open, close): (&Range, &Range)) -> Self { + Self { + open: open.into(), + close: close.into(), + } + } + } + + impl Default for $name { + fn default() -> Self { + Self { + open: Span { + start: 0, + end: 0, + }, + close: Span { + start: 0, + end: 0, + } + } + } + } + + impl Delimiter for $name { + const OPEN: char = $token_open; + const CLOSE: char = $token_close; + + #[must_use] + fn open(&self) -> Span { + self.open + } + + #[must_use] + fn close(&self) -> Span { + self.close + } + + #[must_use] + fn range(&self) -> Span { + Span{ + start: self.open.start, + end: self.close.end, + } + } + } + )* + }; +} + +define_keywords! { + /// The `mod` keyword. + Mod "mod" + /// The `struct` keyword. + Struct "struct" + /// The `enum` keyword. + Enum "enum" + /// The `const` keyword. + Const "const" + /// The `type` keyword. + Type "type" + /// The `use` keyword. + Use "use" +} + +define_punctuation! { + /// Comma `,` separator, usually used to separate fields or enum variants. + Comma "," + /// Colon `:` separator, as separator between field names and their type. + Colon ":" + /// Semicolon `;` separator, as terminator for type aliases. + Semicolon ";" + /// Pound `#` punctuation, as start for an attribute block. + Pound "#" + /// Double colon `::` separator, as path separator for type paths. + DoubleColon "::" + /// Equal sign `=` separator, used in type aliases. + Equal "=" +} + +define_delimiters! { + /// Braces `{`...`}`, to surround module content and named fields. + Brace '{' '}' + /// Brackets `[`...`]`, to surround byte array literals. + Bracket '[' ']' + /// Parenthesis `(`...`)`, to surround unnamed fields and tuples. + Parenthesis '(' ')' + /// Angles `<`...`>`, to surround generics. + Angle '<' '>' +} diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@alias_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@alias_basic.mabo.snap index 4c46118..7840fd4 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@alias_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@alias_basic.mabo.snap @@ -21,15 +21,18 @@ Schema { }, ], ), + keyword: Type, name: Name { value: "Sample", }, generics: Generics( [], ), + equal: Equal, target: Type { value: U32, }, + semicolon: Semicolon, }, ), ], diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@attribute_multi.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@attribute_multi.mabo.snap index d55b1c3..cacee57 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@attribute_multi.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@attribute_multi.mabo.snap @@ -48,6 +48,7 @@ Schema { }, ], ), + keyword: Struct, name: Name { value: "Sample", }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@attribute_single.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@attribute_single.mabo.snap index 9fec2bb..e9ba475 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@attribute_single.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@attribute_single.mabo.snap @@ -31,6 +31,7 @@ Schema { }, ], ), + keyword: Struct, name: Name { value: "Sample", }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@attribute_unit.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@attribute_unit.mabo.snap index 0dc6bc1..9cd988f 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@attribute_unit.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@attribute_unit.mabo.snap @@ -25,6 +25,7 @@ Schema { }, ], ), + keyword: Struct, name: Name { value: "Sample", }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@attributes.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@attributes.mabo.snap index 06e9d1f..054964e 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@attributes.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@attributes.mabo.snap @@ -73,6 +73,7 @@ Schema { }, ], ), + keyword: Struct, name: Name { value: "Sample", }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@attributes_min_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@attributes_min_ws.mabo.snap index 787317d..d3d81a0 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@attributes_min_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@attributes_min_ws.mabo.snap @@ -73,6 +73,7 @@ Schema { }, ], ), + keyword: Struct, name: Name { value: "Sample", }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@const_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@const_basic.mabo.snap index c0f86cb..207e36b 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@const_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@const_basic.mabo.snap @@ -17,17 +17,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "BOOL_TRUE", }, + colon: Colon, ty: Type { value: Bool, }, + equal: Equal, value: Literal { value: Bool( true, ), }, + semicolon: Semicolon, }, ), Const( @@ -35,17 +39,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "BOOL_FALSE", }, + colon: Colon, ty: Type { value: Bool, }, + equal: Equal, value: Literal { value: Bool( false, ), }, + semicolon: Semicolon, }, ), Const( @@ -53,17 +61,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "INT", }, + colon: Colon, ty: Type { value: U32, }, + equal: Equal, value: Literal { value: Int( 100, ), }, + semicolon: Semicolon, }, ), Const( @@ -71,17 +83,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "FLOAT", }, + colon: Colon, ty: Type { value: F64, }, + equal: Equal, value: Literal { value: Float( 5.5, ), }, + semicolon: Semicolon, }, ), Const( @@ -89,17 +105,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "STRING", }, + colon: Colon, ty: Type { value: String, }, + equal: Equal, value: Literal { value: String( "value", ), }, + semicolon: Semicolon, }, ), Const( @@ -107,12 +127,15 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "BYTES", }, + colon: Colon, ty: Type { value: Bytes, }, + equal: Equal, value: Literal { value: Bytes( [ @@ -122,6 +145,7 @@ Schema { ], ), }, + semicolon: Semicolon, }, ), ], diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@const_string.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@const_string.mabo.snap index dbde4d2..83bc943 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@const_string.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@const_string.mabo.snap @@ -17,17 +17,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "SIMPLE", }, + colon: Colon, ty: Type { value: String, }, + equal: Equal, value: Literal { value: String( "value", ), }, + semicolon: Semicolon, }, ), Const( @@ -35,17 +39,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "NEWLINE_ESCAPE", }, + colon: Colon, ty: Type { value: String, }, + equal: Equal, value: Literal { value: String( "one two three", ), }, + semicolon: Semicolon, }, ), Const( @@ -53,17 +61,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "ESCAPES", }, + colon: Colon, ty: Type { value: String, }, + equal: Equal, value: Literal { value: String( "escape basics \r\n \t \u{8} \u{c} \\ \"hello\" \nunicode ❤ emoji ❤ ", ), }, + semicolon: Semicolon, }, ), Const( @@ -71,17 +83,21 @@ Schema { comment: Comment( [], ), + keyword: Const, name: Name { value: "MULTILINE", }, + colon: Colon, ty: Type { value: String, }, + equal: Equal, value: Literal { value: String( "a\n b\n c\n", ), }, + semicolon: Semicolon, }, ), ], diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@enum_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@enum_basic.mabo.snap index 42bfc5d..71733d0 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@enum_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@enum_basic.mabo.snap @@ -24,12 +24,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Sample", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -44,6 +46,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -57,6 +62,7 @@ Schema { value: "Two", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -67,6 +73,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -77,6 +86,7 @@ Schema { value: 2, }, ), + comma: None, }, ], ), @@ -85,6 +95,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -94,6 +107,7 @@ Schema { value: "Three", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -102,6 +116,7 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: U32, }, @@ -110,6 +125,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -122,6 +140,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: Bool, }, @@ -130,6 +149,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -138,6 +160,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@enum_generics.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@enum_generics.mabo.snap index 2412153..cd2d30b 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@enum_generics.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@enum_generics.mabo.snap @@ -24,6 +24,7 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Sample", }, @@ -43,6 +44,7 @@ Schema { }, ], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -57,6 +59,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -66,6 +71,7 @@ Schema { value: "Two", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -75,6 +81,7 @@ Schema { name: Name { value: "A", }, + angle: None, generics: [], }, ), @@ -84,6 +91,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -93,6 +103,7 @@ Schema { name: Name { value: "B", }, + angle: None, generics: [], }, ), @@ -102,6 +113,7 @@ Schema { value: 2, }, ), + comma: None, }, ], ), @@ -110,6 +122,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -119,6 +134,7 @@ Schema { value: "Three", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -127,6 +143,7 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -134,6 +151,7 @@ Schema { name: Name { value: "C", }, + angle: None, generics: [], }, ), @@ -143,6 +161,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -151,6 +172,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -158,6 +180,7 @@ Schema { name: Name { value: "D", }, + angle: None, generics: [], }, ), @@ -167,6 +190,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -175,6 +201,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@enum_many_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@enum_many_ws.mabo.snap index d76a13b..7988aef 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@enum_many_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@enum_many_ws.mabo.snap @@ -24,12 +24,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Sample", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -44,6 +46,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -53,6 +58,7 @@ Schema { value: "Two", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -63,6 +69,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -73,6 +82,7 @@ Schema { value: 2, }, ), + comma: None, }, ], ), @@ -81,6 +91,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -90,6 +103,7 @@ Schema { value: "Three", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -98,6 +112,7 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: U32, }, @@ -106,6 +121,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -114,6 +132,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: Bool, }, @@ -122,6 +141,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -130,6 +152,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@enum_min_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@enum_min_ws.mabo.snap index 61ce229..5194661 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@enum_min_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@enum_min_ws.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Sample", }, @@ -30,6 +31,7 @@ Schema { }, ], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -44,6 +46,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -53,6 +58,7 @@ Schema { value: "Two", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -63,6 +69,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -73,6 +82,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -82,6 +94,7 @@ Schema { name: Name { value: "T", }, + angle: None, generics: [], }, ), @@ -91,6 +104,7 @@ Schema { value: 3, }, ), + comma: None, }, ], ), @@ -99,6 +113,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -108,6 +125,7 @@ Schema { value: "Three", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -116,6 +134,7 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: U32, }, @@ -124,6 +143,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -132,6 +154,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: Bool, }, @@ -140,6 +163,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -148,6 +174,7 @@ Schema { name: Name { value: "field3", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -155,6 +182,7 @@ Schema { name: Name { value: "T", }, + angle: None, generics: [], }, ), @@ -164,6 +192,7 @@ Schema { value: 3, }, ), + comma: None, }, ], ), @@ -172,6 +201,7 @@ Schema { value: 3, }, ), + comma: None, }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@import_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@import_basic.mabo.snap index a23de4a..c17a0f3 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@import_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@import_basic.mabo.snap @@ -14,6 +14,7 @@ Schema { definitions: [ Import( Import { + keyword: Use, full: Name { value: "other::schema::Sample", }, @@ -30,10 +31,12 @@ Schema { value: "Sample", }, ), + semicolon: Semicolon, }, ), Import( Import { + keyword: Use, full: Name { value: "second::submodule", }, @@ -46,6 +49,7 @@ Schema { }, ], element: None, + semicolon: Semicolon, }, ), ], diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@mixed.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@mixed.mabo.snap index d89ca5b..3d77fee 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@mixed.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@mixed.mabo.snap @@ -37,6 +37,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "User", }, @@ -44,6 +45,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -52,6 +54,7 @@ Schema { name: Name { value: "name", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -59,6 +62,7 @@ Schema { name: Name { value: "FullName", }, + angle: None, generics: [], }, ), @@ -68,6 +72,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -80,8 +87,10 @@ Schema { name: Name { value: "address", }, + colon: Colon, ty: Type { value: Option( + Angle, Type { value: External( ExternalType { @@ -89,6 +98,7 @@ Schema { name: Name { value: "Address", }, + angle: None, generics: [], }, ), @@ -100,6 +110,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -108,6 +121,7 @@ Schema { name: Name { value: "age", }, + colon: Colon, ty: Type { value: U8, }, @@ -116,6 +130,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -124,6 +141,7 @@ Schema { name: Name { value: "birthday", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -135,6 +153,7 @@ Schema { name: Name { value: "DayOfBirth", }, + angle: None, generics: [], }, ), @@ -144,6 +163,9 @@ Schema { value: 4, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -161,6 +183,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "FullName", }, @@ -168,6 +191,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -176,6 +200,7 @@ Schema { name: Name { value: "first", }, + colon: Colon, ty: Type { value: String, }, @@ -184,6 +209,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -192,8 +220,10 @@ Schema { name: Name { value: "middle", }, + colon: Colon, ty: Type { value: Option( + Angle, Type { value: String, }, @@ -204,6 +234,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -212,6 +245,7 @@ Schema { name: Name { value: "last", }, + colon: Colon, ty: Type { value: String, }, @@ -220,6 +254,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -243,12 +280,14 @@ Schema { }, ], ), + keyword: Type, name: Name { value: "Name", }, generics: Generics( [], ), + equal: Equal, target: Type { value: External( ExternalType { @@ -256,10 +295,12 @@ Schema { name: Name { value: "FullName", }, + angle: None, generics: [], }, ), }, + semicolon: Semicolon, }, ), Struct( @@ -270,6 +311,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Address", }, @@ -277,6 +319,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -289,6 +332,7 @@ Schema { name: Name { value: "street", }, + colon: Colon, ty: Type { value: String, }, @@ -297,6 +341,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -309,6 +356,7 @@ Schema { name: Name { value: "house_no", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -316,6 +364,7 @@ Schema { name: Name { value: "HouseNumber", }, + angle: None, generics: [], }, ), @@ -325,6 +374,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -333,6 +385,7 @@ Schema { name: Name { value: "city", }, + colon: Colon, ty: Type { value: String, }, @@ -341,6 +394,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -364,12 +420,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "HouseNumber", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -383,6 +441,7 @@ Schema { value: "Digit", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -393,6 +452,7 @@ Schema { value: 1, }, ), + comma: None, }, ], ), @@ -401,6 +461,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -414,6 +477,7 @@ Schema { value: "Text", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -424,6 +488,7 @@ Schema { value: 1, }, ), + comma: None, }, ], ), @@ -432,6 +497,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], }, @@ -445,17 +513,21 @@ Schema { }, ], ), + keyword: Const, name: Name { value: "MAX_AGE", }, + colon: Colon, ty: Type { value: U8, }, + equal: Equal, value: Literal { value: Int( 120, ), }, + semicolon: Semicolon, }, ), Module( @@ -467,9 +539,11 @@ Schema { }, ], ), + keyword: Mod, name: Name { value: "birthday", }, + brace: Brace, definitions: [ Enum( Enum { @@ -483,12 +557,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "DayOfBirth", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -498,6 +574,7 @@ Schema { value: "Specific", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -506,6 +583,7 @@ Schema { name: Name { value: "year", }, + colon: Colon, ty: Type { value: U16, }, @@ -514,6 +592,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -522,6 +603,7 @@ Schema { name: Name { value: "month", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -529,6 +611,7 @@ Schema { name: Name { value: "Month", }, + angle: None, generics: [], }, ), @@ -538,6 +621,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -546,6 +632,7 @@ Schema { name: Name { value: "day", }, + colon: Colon, ty: Type { value: U8, }, @@ -554,6 +641,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -562,6 +652,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -575,6 +668,7 @@ Schema { value: "Secret", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -590,8 +684,10 @@ Schema { name: Name { value: "reason", }, + colon: Colon, ty: Type { value: Option( + Angle, Type { value: String, }, @@ -602,6 +698,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -610,6 +709,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -628,6 +730,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], }, @@ -641,17 +746,21 @@ Schema { }, ], ), + keyword: Const, name: Name { value: "MIN_YEAR", }, + colon: Colon, ty: Type { value: U16, }, + equal: Equal, value: Literal { value: Int( 1900, ), }, + semicolon: Semicolon, }, ), Const( @@ -666,17 +775,21 @@ Schema { }, ], ), + keyword: Const, name: Name { value: "MAX_DAY", }, + colon: Colon, ty: Type { value: U8, }, + equal: Equal, value: Literal { value: Int( 31, ), }, + semicolon: Semicolon, }, ), Enum( @@ -691,12 +804,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Month", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -711,6 +826,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -725,6 +843,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -739,6 +860,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -753,6 +877,9 @@ Schema { value: 4, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -767,6 +894,9 @@ Schema { value: 5, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -781,6 +911,9 @@ Schema { value: 6, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -795,6 +928,9 @@ Schema { value: 7, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -809,6 +945,9 @@ Schema { value: 8, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -823,6 +962,9 @@ Schema { value: 9, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -837,6 +979,9 @@ Schema { value: 10, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -851,6 +996,9 @@ Schema { value: 11, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -865,6 +1013,9 @@ Schema { value: 12, }, ), + comma: Some( + Comma, + ), }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@module_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@module_basic.mabo.snap index 7bbdd33..16aee3a 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@module_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@module_basic.mabo.snap @@ -17,9 +17,11 @@ Schema { comment: Comment( [], ), + keyword: Mod, name: Name { value: "a", }, + brace: Brace, definitions: [ Module( Module { @@ -30,9 +32,11 @@ Schema { }, ], ), + keyword: Mod, name: Name { value: "b", }, + brace: Brace, definitions: [ Enum( Enum { @@ -42,12 +46,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Sample", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -62,6 +68,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, ], }, @@ -77,6 +86,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -84,6 +94,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -92,6 +103,7 @@ Schema { name: Name { value: "value", }, + colon: Colon, ty: Type { value: U32, }, @@ -100,6 +112,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -108,6 +123,7 @@ Schema { name: Name { value: "inner", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -119,6 +135,7 @@ Schema { name: Name { value: "Sample", }, + angle: None, generics: [], }, ), @@ -128,6 +145,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@optional_ids.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@optional_ids.mabo.snap index 3ad58bc..36d3bfc 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@optional_ids.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@optional_ids.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "SampleNamed", }, @@ -27,6 +28,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -35,10 +37,14 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: U32, }, id: None, + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -47,6 +53,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: U32, }, @@ -55,6 +62,9 @@ Schema { value: 100, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -63,10 +73,14 @@ Schema { name: Name { value: "field3", }, + colon: Colon, ty: Type { value: U32, }, id: None, + comma: Some( + Comma, + ), }, ], ), @@ -80,6 +94,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "SampleUnnamed", }, @@ -87,12 +102,16 @@ Schema { [], ), fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { value: U32, }, id: None, + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -103,12 +122,16 @@ Schema { value: 100, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { value: U32, }, id: None, + comma: None, }, ], ), @@ -122,12 +145,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "SampleEnum", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -137,6 +162,7 @@ Schema { value: "Named", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -145,10 +171,14 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: U32, }, id: None, + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -157,6 +187,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: U32, }, @@ -165,6 +196,9 @@ Schema { value: 100, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -173,14 +207,21 @@ Schema { name: Name { value: "field3", }, + colon: Colon, ty: Type { value: U32, }, id: None, + comma: Some( + Comma, + ), }, ], ), id: None, + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -195,6 +236,9 @@ Schema { value: 50, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -204,12 +248,16 @@ Schema { value: "Unnamed", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { value: U32, }, id: None, + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -220,16 +268,23 @@ Schema { value: 100, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { value: U32, }, id: None, + comma: None, }, ], ), id: None, + comma: Some( + Comma, + ), }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@schema_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@schema_basic.mabo.snap index 7310891..6c7446d 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@schema_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@schema_basic.mabo.snap @@ -24,6 +24,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "SampleStruct", }, @@ -31,6 +32,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -39,6 +41,7 @@ Schema { name: Name { value: "a", }, + colon: Colon, ty: Type { value: U32, }, @@ -47,6 +50,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -55,6 +61,7 @@ Schema { name: Name { value: "b", }, + colon: Colon, ty: Type { value: Bool, }, @@ -63,6 +70,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -80,12 +90,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "SampleEnum", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -100,6 +112,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -109,6 +124,7 @@ Schema { value: "Two", }, fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -119,6 +135,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -129,6 +148,7 @@ Schema { value: 2, }, ), + comma: None, }, ], ), @@ -137,6 +157,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, Variant { comment: Comment( @@ -146,6 +169,7 @@ Schema { value: "Three", }, fields: Named( + Brace, [ NamedField { comment: Comment( @@ -154,6 +178,7 @@ Schema { name: Name { value: "field1", }, + colon: Colon, ty: Type { value: U32, }, @@ -162,6 +187,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -170,6 +198,7 @@ Schema { name: Name { value: "field2", }, + colon: Colon, ty: Type { value: Bool, }, @@ -178,6 +207,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -186,6 +218,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], }, diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@struct_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@struct_basic.mabo.snap index 576c185..c0c0b24 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@struct_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@struct_basic.mabo.snap @@ -24,6 +24,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -31,6 +32,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -39,6 +41,7 @@ Schema { name: Name { value: "a", }, + colon: Colon, ty: Type { value: U32, }, @@ -47,6 +50,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -59,6 +65,7 @@ Schema { name: Name { value: "b", }, + colon: Colon, ty: Type { value: Bool, }, @@ -67,6 +74,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@struct_generics.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@struct_generics.mabo.snap index e6d0fe3..8ec89ad 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@struct_generics.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@struct_generics.mabo.snap @@ -24,6 +24,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "KeyValue", }, @@ -38,6 +39,7 @@ Schema { ], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -46,6 +48,7 @@ Schema { name: Name { value: "key", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -53,6 +56,7 @@ Schema { name: Name { value: "K", }, + angle: None, generics: [], }, ), @@ -62,6 +66,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -70,6 +77,7 @@ Schema { name: Name { value: "value", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -77,6 +85,7 @@ Schema { name: Name { value: "V", }, + angle: None, generics: [], }, ), @@ -86,6 +95,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@struct_many_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@struct_many_ws.mabo.snap index d33ecab..2cc99f0 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@struct_many_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@struct_many_ws.mabo.snap @@ -24,6 +24,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -35,6 +36,7 @@ Schema { ], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -43,6 +45,7 @@ Schema { name: Name { value: "a", }, + colon: Colon, ty: Type { value: U32, }, @@ -51,6 +54,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -59,6 +65,7 @@ Schema { name: Name { value: "b", }, + colon: Colon, ty: Type { value: Bool, }, @@ -67,6 +74,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -75,6 +85,7 @@ Schema { name: Name { value: "c", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -82,6 +93,7 @@ Schema { name: Name { value: "T", }, + angle: None, generics: [], }, ), @@ -91,6 +103,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@struct_min_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@struct_min_ws.mabo.snap index 0220366..090c6b0 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@struct_min_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@struct_min_ws.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -31,6 +32,7 @@ Schema { ], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -39,6 +41,7 @@ Schema { name: Name { value: "a", }, + colon: Colon, ty: Type { value: U32, }, @@ -47,6 +50,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -55,6 +61,7 @@ Schema { name: Name { value: "b", }, + colon: Colon, ty: Type { value: Bool, }, @@ -63,6 +70,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -71,6 +81,7 @@ Schema { name: Name { value: "c", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -78,6 +89,7 @@ Schema { name: Name { value: "T", }, + angle: None, generics: [], }, ), @@ -87,6 +99,7 @@ Schema { value: 3, }, ), + comma: None, }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@struct_tuple.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@struct_tuple.mabo.snap index 1c60edb..f8f7d79 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@struct_tuple.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@struct_tuple.mabo.snap @@ -24,6 +24,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -31,6 +32,7 @@ Schema { [], ), fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { @@ -41,6 +43,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { @@ -51,6 +56,7 @@ Schema { value: 2, }, ), + comma: None, }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@types_basic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@types_basic.mabo.snap index a44b83a..38e6664 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@types_basic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@types_basic.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -27,6 +28,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -35,6 +37,7 @@ Schema { name: Name { value: "f01", }, + colon: Colon, ty: Type { value: Bool, }, @@ -43,6 +46,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -51,6 +57,7 @@ Schema { name: Name { value: "f02", }, + colon: Colon, ty: Type { value: U8, }, @@ -59,6 +66,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -67,6 +77,7 @@ Schema { name: Name { value: "f03", }, + colon: Colon, ty: Type { value: U16, }, @@ -75,6 +86,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -83,6 +97,7 @@ Schema { name: Name { value: "f04", }, + colon: Colon, ty: Type { value: U32, }, @@ -91,6 +106,9 @@ Schema { value: 4, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -99,6 +117,7 @@ Schema { name: Name { value: "f05", }, + colon: Colon, ty: Type { value: U64, }, @@ -107,6 +126,9 @@ Schema { value: 5, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -115,6 +137,7 @@ Schema { name: Name { value: "f06", }, + colon: Colon, ty: Type { value: U128, }, @@ -123,6 +146,9 @@ Schema { value: 6, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -131,6 +157,7 @@ Schema { name: Name { value: "f07", }, + colon: Colon, ty: Type { value: I8, }, @@ -139,6 +166,9 @@ Schema { value: 7, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -147,6 +177,7 @@ Schema { name: Name { value: "f08", }, + colon: Colon, ty: Type { value: I16, }, @@ -155,6 +186,9 @@ Schema { value: 8, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -163,6 +197,7 @@ Schema { name: Name { value: "f09", }, + colon: Colon, ty: Type { value: I32, }, @@ -171,6 +206,9 @@ Schema { value: 9, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -179,6 +217,7 @@ Schema { name: Name { value: "f10", }, + colon: Colon, ty: Type { value: I64, }, @@ -187,6 +226,9 @@ Schema { value: 10, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -195,6 +237,7 @@ Schema { name: Name { value: "f11", }, + colon: Colon, ty: Type { value: I128, }, @@ -203,6 +246,9 @@ Schema { value: 11, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -211,6 +257,7 @@ Schema { name: Name { value: "f12", }, + colon: Colon, ty: Type { value: F32, }, @@ -219,6 +266,9 @@ Schema { value: 12, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -227,6 +277,7 @@ Schema { name: Name { value: "f13", }, + colon: Colon, ty: Type { value: F64, }, @@ -235,6 +286,9 @@ Schema { value: 13, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -243,6 +297,7 @@ Schema { name: Name { value: "f14", }, + colon: Colon, ty: Type { value: String, }, @@ -251,6 +306,9 @@ Schema { value: 14, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -259,6 +317,7 @@ Schema { name: Name { value: "f15", }, + colon: Colon, ty: Type { value: StringRef, }, @@ -267,6 +326,9 @@ Schema { value: 15, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -275,6 +337,7 @@ Schema { name: Name { value: "f16", }, + colon: Colon, ty: Type { value: Bytes, }, @@ -283,6 +346,9 @@ Schema { value: 16, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -291,6 +357,7 @@ Schema { name: Name { value: "f17", }, + colon: Colon, ty: Type { value: BytesRef, }, @@ -299,6 +366,9 @@ Schema { value: 17, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -307,6 +377,7 @@ Schema { name: Name { value: "f18", }, + colon: Colon, ty: Type { value: BoxString, }, @@ -315,6 +386,9 @@ Schema { value: 18, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -323,6 +397,7 @@ Schema { name: Name { value: "f19", }, + colon: Colon, ty: Type { value: BoxBytes, }, @@ -331,6 +406,9 @@ Schema { value: 19, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -339,8 +417,10 @@ Schema { name: Name { value: "f20", }, + colon: Colon, ty: Type { value: Tuple( + Parenthesis, [ Type { value: U32, @@ -359,6 +439,9 @@ Schema { value: 20, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -367,11 +450,14 @@ Schema { name: Name { value: "f21", }, + colon: Colon, ty: Type { value: Array( + Bracket, Type { value: U32, }, + Semicolon, 12, ), }, @@ -380,6 +466,9 @@ Schema { value: 21, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@types_generic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@types_generic.mabo.snap index d10093c..94800f6 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@types_generic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@types_generic.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -27,6 +28,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -35,8 +37,10 @@ Schema { name: Name { value: "f1", }, + colon: Colon, ty: Type { value: Vec( + Angle, Type { value: U32, }, @@ -47,6 +51,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -55,8 +62,11 @@ Schema { name: Name { value: "f2", }, + colon: Colon, ty: Type { value: HashMap( + Angle, + Comma, ( Type { value: U32, @@ -72,6 +82,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -80,8 +93,10 @@ Schema { name: Name { value: "f3", }, + colon: Colon, ty: Type { value: HashSet( + Angle, Type { value: U32, }, @@ -92,6 +107,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -100,8 +118,10 @@ Schema { name: Name { value: "f4", }, + colon: Colon, ty: Type { value: Option( + Angle, Type { value: U32, }, @@ -112,6 +132,9 @@ Schema { value: 4, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -120,8 +143,10 @@ Schema { name: Name { value: "f5", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: U32, }, @@ -132,6 +157,9 @@ Schema { value: 5, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -145,6 +173,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "SampleUnnamed", }, @@ -152,10 +181,12 @@ Schema { [], ), fields: Unnamed( + Parenthesis, [ UnnamedField { ty: Type { value: Vec( + Angle, Type { value: U32, }, @@ -166,10 +197,15 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { value: HashMap( + Angle, + Comma, ( Type { value: U32, @@ -185,10 +221,14 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { value: HashSet( + Angle, Type { value: U32, }, @@ -199,10 +239,14 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { value: Option( + Angle, Type { value: U32, }, @@ -213,10 +257,14 @@ Schema { value: 4, }, ), + comma: Some( + Comma, + ), }, UnnamedField { ty: Type { value: NonZero( + Angle, Type { value: U32, }, @@ -227,6 +275,9 @@ Schema { value: 5, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@types_nested.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@types_nested.mabo.snap index 83f2cdb..5dc02ef 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@types_nested.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@types_nested.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -27,6 +28,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -35,14 +37,20 @@ Schema { name: Name { value: "value", }, + colon: Colon, ty: Type { value: Vec( + Angle, Type { value: Option( + Angle, Type { value: NonZero( + Angle, Type { value: HashMap( + Angle, + Comma, ( Type { value: I64, @@ -64,6 +72,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@types_non_zero.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@types_non_zero.mabo.snap index fc6286b..a787c05 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@types_non_zero.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@types_non_zero.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -27,6 +28,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -35,8 +37,10 @@ Schema { name: Name { value: "f01", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: U8, }, @@ -47,6 +51,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -55,8 +62,10 @@ Schema { name: Name { value: "f02", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: U16, }, @@ -67,6 +76,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -75,8 +87,10 @@ Schema { name: Name { value: "f03", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: U32, }, @@ -87,6 +101,9 @@ Schema { value: 3, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -95,8 +112,10 @@ Schema { name: Name { value: "f04", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: U64, }, @@ -107,6 +126,9 @@ Schema { value: 4, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -115,8 +137,10 @@ Schema { name: Name { value: "f05", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: U128, }, @@ -127,6 +151,9 @@ Schema { value: 5, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -135,8 +162,10 @@ Schema { name: Name { value: "f06", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: I8, }, @@ -147,6 +176,9 @@ Schema { value: 6, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -155,8 +187,10 @@ Schema { name: Name { value: "f07", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: I16, }, @@ -167,6 +201,9 @@ Schema { value: 7, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -175,8 +212,10 @@ Schema { name: Name { value: "f08", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: I32, }, @@ -187,6 +226,9 @@ Schema { value: 8, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -195,8 +237,10 @@ Schema { name: Name { value: "f09", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: I64, }, @@ -207,6 +251,9 @@ Schema { value: 9, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -215,8 +262,10 @@ Schema { name: Name { value: "f10", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: I128, }, @@ -227,6 +276,9 @@ Schema { value: 10, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -235,8 +287,10 @@ Schema { name: Name { value: "f11", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: String, }, @@ -247,6 +301,9 @@ Schema { value: 11, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -255,8 +312,10 @@ Schema { name: Name { value: "f12", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: Bytes, }, @@ -267,6 +326,9 @@ Schema { value: 12, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -275,10 +337,13 @@ Schema { name: Name { value: "f13", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: Vec( + Angle, Type { value: String, }, @@ -291,6 +356,9 @@ Schema { value: 13, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -299,10 +367,14 @@ Schema { name: Name { value: "f14", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: HashMap( + Angle, + Comma, ( Type { value: String, @@ -320,6 +392,9 @@ Schema { value: 14, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -328,10 +403,13 @@ Schema { name: Name { value: "f15", }, + colon: Colon, ty: Type { value: NonZero( + Angle, Type { value: HashSet( + Angle, Type { value: String, }, @@ -344,6 +422,9 @@ Schema { value: 15, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__parse@types_ref.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__parse@types_ref.mabo.snap index 763d4dc..ded432b 100644 --- a/crates/mabo-parser/tests/snapshots/parser__parse@types_ref.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__parse@types_ref.mabo.snap @@ -20,6 +20,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "Sample", }, @@ -27,6 +28,7 @@ Schema { [], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -35,6 +37,7 @@ Schema { name: Name { value: "basic", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -42,6 +45,7 @@ Schema { name: Name { value: "Test123", }, + angle: None, generics: [], }, ), @@ -51,6 +55,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -59,6 +66,7 @@ Schema { name: Name { value: "with_generics", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -66,6 +74,9 @@ Schema { name: Name { value: "KeyValue", }, + angle: Some( + Angle, + ), generics: [ Type { value: U32, @@ -82,6 +93,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), @@ -95,12 +109,14 @@ Schema { attributes: Attributes( [], ), + keyword: Enum, name: Name { value: "Test123", }, generics: Generics( [], ), + brace: Brace, variants: [ Variant { comment: Comment( @@ -115,6 +131,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, ], }, @@ -127,6 +146,7 @@ Schema { attributes: Attributes( [], ), + keyword: Struct, name: Name { value: "KeyValue", }, @@ -141,6 +161,7 @@ Schema { ], ), fields: Named( + Brace, [ NamedField { comment: Comment( @@ -149,6 +170,7 @@ Schema { name: Name { value: "key", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -156,6 +178,7 @@ Schema { name: Name { value: "K", }, + angle: None, generics: [], }, ), @@ -165,6 +188,9 @@ Schema { value: 1, }, ), + comma: Some( + Comma, + ), }, NamedField { comment: Comment( @@ -173,6 +199,7 @@ Schema { name: Name { value: "value", }, + colon: Colon, ty: Type { value: External( ExternalType { @@ -180,6 +207,7 @@ Schema { name: Name { value: "V", }, + angle: None, generics: [], }, ), @@ -189,6 +217,9 @@ Schema { value: 2, }, ), + comma: Some( + Comma, + ), }, ], ), diff --git a/crates/mabo-parser/tests/snapshots/parser__print@enum_min_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__print@enum_min_ws.mabo.snap index 5a2115e..e4b1f31 100644 --- a/crates/mabo-parser/tests/snapshots/parser__print@enum_min_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__print@enum_min_ws.mabo.snap @@ -9,8 +9,8 @@ enum Sample { Three { field1: u32 @1, field2: bool @2, - field3: T @3, - } @3, + field3: T @3 + } @3 } diff --git a/crates/mabo-parser/tests/snapshots/parser__print@struct_min_ws.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__print@struct_min_ws.mabo.snap index bad5c18..4a4024f 100644 --- a/crates/mabo-parser/tests/snapshots/parser__print@struct_min_ws.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__print@struct_min_ws.mabo.snap @@ -6,7 +6,7 @@ input_file: crates/mabo-parser/tests/inputs/struct_min_ws.mabo struct Sample { a: u32 @1, b: bool @2, - c: T @3, + c: T @3 } diff --git a/crates/mabo-parser/tests/snapshots/parser__print@types_generic.mabo.snap b/crates/mabo-parser/tests/snapshots/parser__print@types_generic.mabo.snap index 6d5a760..6829661 100644 --- a/crates/mabo-parser/tests/snapshots/parser__print@types_generic.mabo.snap +++ b/crates/mabo-parser/tests/snapshots/parser__print@types_generic.mabo.snap @@ -11,6 +11,6 @@ struct Sample { f5: non_zero @5, } -struct SampleUnnamed(vec @1, hash_map @2, hash_set @3, option @4, non_zero @5) +struct SampleUnnamed(vec @1, hash_map @2, hash_set @3, option @4, non_zero @5,) diff --git a/vscode-extension/package.json b/vscode-extension/package.json index 5ec093b..dfc56a1 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -114,6 +114,47 @@ "id": "typeAlias", "description": "Style for type aliases", "superType": "type" + }, + { + "id": "comma", + "description": "Style for comma punctuation" + }, + { + "id": "colon", + "description": "Style for colon punctuation" + }, + { + "id": "semicolon", + "description": "Style for comsemicolonma punctuation" + }, + { + "id": "pound", + "description": "Style for pound punctuation" + }, + { + "id": "doubleColon", + "description": "Style for double colon punctuation" + }, + { + "id": "equal", + "description": "Style for equal sign punctuation", + "superType": "operator" + }, + { + "id": "brace", + "description": "Style for brace delimiters" + }, + { + "id": "bracket", + "description": "Style for bracket delimiters" + }, + { + "id": "parenthesis", + "description": "Style for parenthesis delimiters" + }, + { + "id": "angle", + "description": "Style for angle delimiters" } ], "semanticTokenModifiers": [ @@ -140,6 +181,37 @@ ], "variable.constant": [ "variable.other.constant.mabo" + ], + "comma": [ + "punctuation.comma.mabo" + ], + "colon": [ + "punctuation.colon.mabo" + ], + "semicolon": [ + "punctuation.semicolon.mabo" + ], + "pound": [ + "punctuation.pound.mabo" + ], + "doubleColon": [ + "punctuation.colon.mabo", + "punctuation.doubleColon.mabo" + ], + "equal": [ + "operator.equal.mabo" + ], + "brace": [ + "punctuation.brackets.curly" + ], + "bracket": [ + "punctuation.brackets.square" + ], + "parenthesis": [ + "punctuation.brackets.round" + ], + "angle": [ + "punctuation.brackets.angle" ] } }