Skip to content

Commit

Permalink
refactor: simplify token parsing
Browse files Browse the repository at this point in the history 
Create a `surround` parser for delimiters that performs the common task
of wrapping any parser with an opening and closing delimiter.

Also, create some helper parsers for each token to reduce the amount of
boilerplate wherever the tokens are used together with other parsers.
  • Loading branch information
@dnaka91
dnaka91 committed Feb 21, 2024
1 parent ba12d5e commit 2a9167c
Show file tree
Hide file tree
Showing 14 changed files with 192 additions and 245 deletions.
2 changes: 1 addition & 1 deletion crates/mabo-compiler/src/simplify.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,7 @@ fn simplify_import<'a>(item: &'a mabo_parser::Import<'_>) -> Import<'a> {
element: item
.element
.as_ref()
.map(|(_,element)| element.get().into()),
.map(|(_, element)| element.get().into()),
}
}

Expand Down
29 changes: 23 additions & 6 deletions crates/mabo-parser/src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ use std::ops::Range;

use winnow::{
ascii::{multispace0, newline, space0},
combinator::{fail, opt, peek, preceded, repeat, terminated, trace},
combinator::{cut_err, fail, opt, peek, preceded, repeat, terminated, trace},
dispatch,
error::{ErrMode, ParserError},
prelude::*,
stream::{AsChar, Stream, StreamIsPartial},
stream::{AsChar, Compare, Location, Stream, StreamIsPartial},
token::any,
};

Expand All @@ -29,6 +29,7 @@ use crate::{
error::{ParseDefinitionError, ParseSchemaCause},
ext::ParserExt,
punctuated::Punctuated,
token::Delimiter,
Definition, Schema,
};

Expand Down Expand Up @@ -217,10 +218,9 @@ where
pub fn punctuate<I, O, P, E, F, G>(mut f: F, mut g: G) -> impl Parser<I, Punctuated<O, P>, E>
where
I: Stream,
P: Copy + From<Range<usize>>,
E: ParserError<I>,
F: Parser<I, (O, Range<usize>), E>,
G: Parser<I, (O, Option<Range<usize>>), E>,
F: Parser<I, (O, P), E>,
G: Parser<I, (O, Option<P>), E>,
{
trace("punctuate", move |i: &mut I| {
let mut values = Vec::new();
Expand All @@ -236,7 +236,7 @@ where
return Err(ErrMode::assert(i, "`repeat` parsers must always consume"));
}

values.push((o, range.into()));
values.push((o, range));
start_prev = Some(start);
}
Err(ErrMode::Backtrack(_)) => {
Expand Down Expand Up @@ -267,3 +267,20 @@ where
}
})
}

pub fn surround<I, O, D, E, F>(f: F) -> impl Parser<I, (D, O), E>
where
I: Compare<char> + Location + Stream + StreamIsPartial,
I::Token: Clone + AsChar,
D: Delimiter + From<(Range<usize>, Range<usize>)>,
E: ParserError<I>,
F: Parser<I, O, E>,
{
let mut parser = (D::OPEN.span(), cut_err((f, ws(D::CLOSE.span()))));

trace("surround", move |i: &mut I| {
parser
.parse_next(i)
.map(|(open, (o, close))| ((open, close).into(), o))
})
}
18 changes: 7 additions & 11 deletions crates/mabo-parser/src/parser/aliases.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,7 @@ use winnow::{
};

use super::{generics, types, Input, ParserExt, Result};
use crate::{
highlight,
token::{self, Punctuation},
Comment, Name, TypeAlias,
};
use crate::{highlight, token, Comment, Name, TypeAlias};

/// Encountered an invalid `type` alias declaration.
#[derive(Debug, ParserError)]
Expand Down Expand Up @@ -68,25 +64,25 @@ pub enum Cause {

pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result<TypeAlias<'i>, ParseError> {
(
terminated(token::Type::NAME.span(), space1),
terminated(token::Type::parser(), space1),
cut_err((
parse_name,
opt(generics::parse.map_err(Cause::Generics)),
preceded(space0, token::Equal::VALUE.span()),
preceded(space0, token::Equal::parser()),
preceded(space0, types::parse.map_err(Cause::from)),
preceded(space0, token::Semicolon::VALUE.span()),
preceded(space0, token::Semicolon::parser()),
)),
)
.parse_next(input)
.map(
|(keyword, (name, generics, equal, target, semicolon))| TypeAlias {
comment: Comment::default(),
keyword: keyword.into(),
keyword,
name,
generics,
equal: equal.into(),
equal,
target,
semicolon: semicolon.into(),
semicolon,
},
)
.map_err(|e| {
Expand Down
28 changes: 18 additions & 10 deletions crates/mabo-parser/src/parser/attributes.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@ use winnow::{
};

use super::{literals, ws, Input, ParserExt, Result};
use crate::{highlight, Attribute, AttributeValue, Attributes, Literal};
use crate::{
highlight,
token::{self, Delimiter},
Attribute, AttributeValue, Attributes, Literal,
};

/// Encountered an invalid `#[...]` attribute declaration.
#[derive(Debug, ParserError)]
Expand Down Expand Up @@ -63,17 +67,17 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result<Attributes<'i>, ParseEr

fn parse_attribute<'i>(input: &mut Input<'i>) -> Result<Vec<Attribute<'i>>, Cause> {
preceded(
"#[",
(token::Pound::parser(), token::Bracket::OPEN),
cut_err(terminated(
terminated(
separated(
1..,
ws((parse_name, parse_value)).map(|(name, value)| Attribute { name, value }),
ws(','),
ws(token::Comma::parser()),
),
opt(','),
opt(token::Comma::parser()),
),
ws(']'),
ws(token::Bracket::CLOSE),
)),
)
.parse_next(input)
Expand All @@ -99,24 +103,28 @@ fn parse_value<'i>(input: &mut Input<'i>) -> Result<AttributeValue<'i>, Cause> {

fn parse_multi_value<'i>(input: &mut Input<'i>) -> Result<Vec<Attribute<'i>>, Cause> {
preceded(
'(',
token::Parenthesis::OPEN,
cut_err(terminated(
terminated(
separated(
1..,
ws((parse_name, parse_value)).map(|(name, value)| Attribute { name, value }),
ws(','),
ws(token::Comma::parser()),
),
opt(','),
opt(token::Comma::parser()),
),
ws(')'),
ws(token::Parenthesis::CLOSE),
)),
)
.parse_next(input)
}

fn parse_single_value(input: &mut Input<'_>) -> Result<Literal, Cause> {
preceded((space0, '=', space0), literals::parse.map_err(Cause::from)).parse_next(input)
preceded(
(space0, token::Equal::parser(), space0),
literals::parse.map_err(Cause::from),
)
.parse_next(input)
}

fn parse_unit_value(input: &mut Input<'_>) -> Result<(), Cause> {
Expand Down
23 changes: 9 additions & 14 deletions crates/mabo-parser/src/parser/consts.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,7 @@ use winnow::{
};

use super::{literals, types, Input, ParserExt, Result};
use crate::{
highlight, location,
token::{self, Punctuation},
Comment, Const, Name,
};
use crate::{highlight, location, token, Comment, Const, Name};

/// Encountered an invalid `const` declaration.
#[derive(Debug, ParserError)]
Expand Down Expand Up @@ -83,29 +79,28 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result<Const<'i>, ParseError>
let start = input.checkpoint();

(
terminated(token::Const::NAME.span(), space1),
terminated(token::Const::parser(), space1),
cut_err((
parse_name,
token::Colon::VALUE.span(),
token::Colon::parser(),
preceded(space0, types::parse.map_err(Cause::from)),
preceded(space0, token::Equal::VALUE.span()),
preceded(space0, token::Equal::parser()),
preceded(space0, literals::parse.map_err(Cause::from)),
token::Semicolon::VALUE
.span()
token::Semicolon::parser()
.map_err_loc(|at, ()| Cause::UnexpectedChar { at, expected: ';' }),
)),
)
.parse_next(input)
.map(
|(keyword, (name, colon, ty, equal, value, semicolon))| Const {
comment: Comment::default(),
keyword: keyword.into(),
keyword,
name,
colon: colon.into(),
colon,
ty,
equal: equal.into(),
equal,
value,
semicolon: semicolon.into(),
semicolon,
},
)
.map_err(|e| {
Expand Down
30 changes: 9 additions & 21 deletions crates/mabo-parser/src/parser/enums.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,8 @@ use winnow::{
Parser,
};

use super::{comments, fields, generics, ids, punctuate, ws, Input, ParserExt, Result};
use crate::{
highlight,
punctuated::Punctuated,
token::{self, Delimiter, Punctuation},
Attributes, Comment, Enum, Name, Variant,
};
use super::{comments, fields, generics, ids, punctuate, surround, ws, Input, ParserExt, Result};
use crate::{highlight, punctuated::Punctuated, token, Attributes, Comment, Enum, Name, Variant};

/// Encountered an invalid `enum` declaration.
#[derive(Debug, ParserError)]
Expand Down Expand Up @@ -91,7 +86,7 @@ pub enum Cause {

pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result<Enum<'i>, ParseError> {
(
terminated(token::Enum::NAME.span(), space1),
terminated(token::Enum::parser(), space1),
cut_err((
parse_name,
opt(generics::parse.map_err(Cause::from)),
Expand All @@ -102,7 +97,7 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result<Enum<'i>, ParseError> {
.map(|(keyword, (name, generics, (brace, variants)))| Enum {
comment: Comment::default(),
attributes: Attributes::default(),
keyword: keyword.into(),
keyword,
name,
generics,
brace,
Expand Down Expand Up @@ -132,18 +127,11 @@ pub(super) fn parse_name<'i>(input: &mut Input<'i>) -> Result<Name<'i>, Cause> {
fn parse_variants<'i>(
input: &mut Input<'i>,
) -> Result<(token::Brace, Punctuated<Variant<'i>>), Cause> {
(
token::Brace::OPEN.span(),
cut_err((
punctuate(
(parse_variant, ws(token::Comma::VALUE.span())),
(parse_variant, opt(ws(token::Comma::VALUE.span()))),
),
ws(token::Brace::CLOSE.span()),
)),
)
.parse_next(input)
.map(|(brace_open, (variants, brace_close))| ((brace_open, brace_close).into(), variants))
surround(punctuate(
(parse_variant, ws(token::Comma::parser())),
(parse_variant, opt(ws(token::Comma::parser()))),
))
.parse_next(input)
}

fn parse_variant<'i>(input: &mut Input<'i>) -> Result<Variant<'i>, Cause> {
Expand Down
47 changes: 15 additions & 32 deletions crates/mabo-parser/src/parser/fields.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,17 @@ use std::ops::Range;
use mabo_derive::{ParserError, ParserErrorCause};
use winnow::{
ascii::space0,
combinator::{cut_err, opt, peek, preceded},
combinator::{opt, peek, preceded},
dispatch,
error::ErrorKind,
stream::{Location, Stream},
token::{any, one_of, take_while},
Parser,
};

use super::{comments, ids, punctuate, types, ws, Input, ParserExt, Result};
use super::{comments, ids, punctuate, surround, types, ws, Input, ParserExt, Result};
use crate::{
highlight, location,
punctuated::Punctuated,
token::{self, Delimiter, Punctuation},
Fields, Name, NamedField, UnnamedField,
highlight, location, punctuated::Punctuated, token, Fields, Name, NamedField, UnnamedField,
};

/// Encountered an invalid field declaration.
Expand Down Expand Up @@ -94,35 +91,21 @@ pub(super) fn parse<'i>(input: &mut Input<'i>) -> Result<Fields<'i>, ParseError>
fn parse_named<'i>(
input: &mut Input<'i>,
) -> Result<(token::Brace, Punctuated<NamedField<'i>>), Cause> {
(
token::Brace::OPEN.span(),
cut_err((
punctuate(
(parse_named_field, ws(token::Comma::VALUE.span())),
(parse_named_field, opt(ws(token::Comma::VALUE.span()))),
),
ws(token::Brace::CLOSE.span()),
)),
)
.parse_next(input)
.map(|(brace_open, (fields, brace_close))| ((brace_open, brace_close).into(), fields))
surround(punctuate(
(parse_named_field, ws(token::Comma::parser())),
(parse_named_field, opt(ws(token::Comma::parser()))),
))
.parse_next(input)
}

fn parse_unnamed<'i>(
input: &mut Input<'i>,
) -> Result<(token::Parenthesis, Punctuated<UnnamedField<'i>>), Cause> {
(
token::Parenthesis::OPEN.span(),
cut_err((
punctuate(
(parse_unnamed_field, ws(token::Comma::VALUE.span())),
(parse_unnamed_field, opt(ws(token::Comma::VALUE.span()))),
),
ws(token::Parenthesis::CLOSE.span()),
)),
)
.parse_next(input)
.map(|(paren_open, (fields, paren_close))| ((paren_open, paren_close).into(), fields))
surround(punctuate(
(parse_unnamed_field, ws(token::Comma::parser())),
(parse_unnamed_field, opt(ws(token::Comma::parser()))),
))
.parse_next(input)
}

fn parse_unit(input: &mut Input<'_>) -> Result<(), Cause> {
Expand All @@ -148,7 +131,7 @@ fn parse_named_field<'i>(input: &mut Input<'i>) -> Result<NamedField<'i>, Cause>
ws(comments::parse.map_err(Cause::from)),
(
preceded(space0, parse_field_name),
preceded(space0, token::Colon::VALUE.span()),
preceded(space0, token::Colon::parser()),
preceded(space0, types::parse.map_err(Cause::from)),
opt(preceded(space0, ids::parse.map_err(Cause::from))),
)
Expand All @@ -158,7 +141,7 @@ fn parse_named_field<'i>(input: &mut Input<'i>) -> Result<NamedField<'i>, Cause>
.map(|(comment, ((name, colon, ty, id), span))| NamedField {
comment,
name,
colon: colon.into(),
colon,
ty,
id,
span: span.into(),
Expand Down
Loading

0 comments on commit 2a9167c

Please sign in to comment.