Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bugs fixes And Improvements of MBE #1209

Merged
merged 5 commits into from
Apr 25, 2019
Merged

Bugs fixes And Improvements of MBE #1209

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e4e2338
Fix incorrect repeat sep eating
edwin0cheng Apr 23, 2019
dfab545
Add more information on macro rules fail to parse
edwin0cheng Apr 23, 2019
299d97b
Add handling `token` seperator in mbe
edwin0cheng Apr 24, 2019
c55a2db
Fix more bugs
edwin0cheng Apr 25, 2019
1908819
Use panic instead of unwrap
edwin0cheng Apr 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 8 additions & 2 deletions crates/ra_hir/src/ids.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,14 @@ pub struct MacroDefId(pub(crate) AstId<ast::MacroCall>);
pub(crate) fn macro_def_query(db: &impl DefDatabase, id: MacroDefId) -> Option<Arc<MacroRules>> {
let macro_call = id.0.to_node(db);
let arg = macro_call.token_tree()?;
let (tt, _) = mbe::ast_to_token_tree(arg)?;
let rules = MacroRules::parse(&tt).ok()?;
let (tt, _) = mbe::ast_to_token_tree(arg).or_else(|| {
log::warn!("fail on macro_def to token tree: {:#?}", arg);
None
})?;
let rules = MacroRules::parse(&tt).ok().or_else(|| {
log::warn!("fail on macro_def parse: {:#?}", tt);
None
})?;
Some(Arc::new(rules))
}

Expand Down
1 change: 1 addition & 0 deletions crates/ra_mbe/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ ra_parser = { path = "../ra_parser" }
tt = { path = "../ra_tt", package = "ra_tt" }
itertools = "0.8.0"
rustc-hash = "1.0.0"
smallvec = "0.6.9"
108 changes: 101 additions & 7 deletions crates/ra_mbe/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ mod subtree_source;
mod subtree_parser;

use ra_syntax::SmolStr;
use smallvec::SmallVec;

pub use tt::{Delimiter, Punct};

Expand Down Expand Up @@ -98,11 +99,18 @@ pub(crate) struct Subtree {
pub(crate) token_trees: Vec<TokenTree>,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum Separator {
Literal(tt::Literal),
Ident(tt::Ident),
Puncts(SmallVec<[tt::Punct; 3]>),
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Repeat {
pub(crate) subtree: Subtree,
pub(crate) kind: RepeatKind,
pub(crate) separator: Option<char>,
pub(crate) separator: Option<Separator>,
}

#[derive(Clone, Debug, PartialEq, Eq)]
Expand Down Expand Up @@ -175,8 +183,8 @@ impl_froms!(TokenTree: Leaf, Subtree);
let expansion = rules.expand(&invocation_tt).unwrap();
assert_eq!(
expansion.to_string(),
"impl From < Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree :: Leaf (it)}} \
impl From < Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree :: Subtree (it)}}"
"impl From <Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \
impl From <Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}"
)
}

Expand Down Expand Up @@ -384,7 +392,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
"#,
);

assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}");
assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ;bar ()}");
}

#[test]
Expand Down Expand Up @@ -416,6 +424,18 @@ impl_froms!(TokenTree: Leaf, Subtree);
assert_expansion(&rules, "foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}");
}

#[test]
fn test_match_group_with_multichar_sep() {
let rules = create_rules(
r#"
macro_rules! foo {
(fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} );
}"#,
);

assert_expansion(&rules, "foo! (fn baz {true true} )", "fn baz () -> bool {true &&true}");
}

#[test]
fn test_expand_to_item_list() {
let rules = create_rules(
Expand Down Expand Up @@ -597,7 +617,7 @@ MACRO_ITEMS@[0; 40)
assert_expansion(
&rules,
"foo! { bar::<u8>::baz::<u8> }",
"fn foo () {let a = bar :: < u8 > :: baz :: < u8 > ;}",
"fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}",
);
}

Expand Down Expand Up @@ -891,7 +911,7 @@ MACRO_ITEMS@[0; 40)
}
"#,
);
assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref < 'a > {s : & 'a str}"#);
assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#);
}

#[test]
Expand Down Expand Up @@ -1063,7 +1083,81 @@ macro_rules! int_base {
);

assert_expansion(&rules, r#" int_base!{Binary for isize as usize -> Binary}"#,
"# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt :: Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
"# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
);
}

#[test]
fn test_generate_pattern_iterators() {
// from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs
let rules = create_rules(
r#"
macro_rules! generate_pattern_iterators {
{ double ended; with $(#[$common_stability_attribute:meta])*,
$forward_iterator:ident,
$reverse_iterator:ident, $iterty:ty
} => {
fn foo(){}
}
}
"#,
);

assert_expansion(&rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str )"#,
"fn foo () {}");
}

#[test]
fn test_impl_fn_for_zst() {
// from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs
let rules = create_rules(
r#"
macro_rules! impl_fn_for_zst {
{ $( $( #[$attr: meta] )*
struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
|$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty
$body: block; )+
} => {
fn foo(){}
}
}
"#,
);

assert_expansion(&rules, r#"
impl_fn_for_zst ! {
# [ derive ( Clone ) ]
struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug {
c . escape_debug_ext ( false )
} ;

# [ derive ( Clone ) ]
struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode {
c . escape_unicode ( )
} ;
# [ derive ( Clone ) ]
struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault {
c . escape_default ( )
} ;
}
"#,
"fn foo () {}");
}

#[test]
fn test_impl_nonzero_fmt() {
// from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
let rules = create_rules(
r#"
macro_rules! impl_nonzero_fmt {
( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
fn foo() {}
}
}
"#,
);

assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#,
"fn foo () {}");
}
}
100 changes: 70 additions & 30 deletions crates/ra_mbe/src/mbe_expander.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,10 +179,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
// Enable followiing code when everything is fixed
// At least we can dogfood itself to not stackoverflow
//
// "tt" => {
// let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
// res.inner.insert(text.clone(), Binding::Simple(token.into()));
// }
"tt" => {
let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
res.inner.insert(text.clone(), Binding::Simple(token.into()));
}
"item" => {
let item =
input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone();
Expand All @@ -196,6 +196,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
"literal" => {
let literal =
input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();

res.inner.insert(
text.clone(),
Binding::Simple(tt::Leaf::from(literal).into()),
Expand All @@ -210,7 +211,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
}
}
crate::Leaf::Punct(punct) => {
if input.eat_punct() != Some(punct) {
if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
return Err(ExpandError::UnexpectedToken);
}
}
Expand All @@ -226,18 +227,51 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
// This should be replaced by a propper macro-by-example implementation
let mut limit = 128;
let mut counter = 0;
while let Ok(nested) = match_lhs(subtree, input) {
counter += 1;
limit -= 1;
if limit == 0 {
break;
}
res.push_nested(nested)?;
if let Some(separator) = *separator {
if !input.is_eof() {
if input.eat_punct().map(|p| p.char) != Some(separator) {
return Err(ExpandError::UnexpectedToken);

let mut memento = input.save();

loop {
match match_lhs(subtree, input) {
Ok(nested) => {
counter += 1;
limit -= 1;
if limit == 0 {
break;
}

memento = input.save();
res.push_nested(nested)?;
if counter == 1 {
if let crate::RepeatKind::ZeroOrOne = kind {
break;
}
}

if let Some(separator) = separator {
use crate::Separator::*;

if !input
.eat_seperator()
.map(|sep| match (sep, separator) {
(Ident(ref a), Ident(ref b)) => a.text == b.text,
(Literal(ref a), Literal(ref b)) => a.text == b.text,
(Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
let a_iter = a.iter().map(|a| a.char);
let b_iter = b.iter().map(|b| b.char);
a_iter.eq(b_iter)
}
_ => false,
})
.unwrap_or(false)
{
input.rollback(memento);
break;
}
}
}
Err(_) => {
input.rollback(memento);
break;
}
}
}
Expand All @@ -246,10 +280,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
crate::RepeatKind::OneOrMore if counter == 0 => {
return Err(ExpandError::UnexpectedToken);
}
crate::RepeatKind::ZeroOrOne if counter > 1 => {
return Err(ExpandError::UnexpectedToken);
}

_ => {}
}
}
Expand Down Expand Up @@ -314,7 +344,7 @@ fn expand_tt(
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a propper macro-by-example implementation
let mut limit = 128;
let mut has_sep = false;
let mut has_seps = 0;

while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
limit -= 1;
Expand All @@ -325,18 +355,28 @@ fn expand_tt(
nesting.push(idx + 1);
token_trees.push(reduce_single_token(t).into());

if let Some(sep) = repeat.separator {
let punct =
tt::Leaf::from(tt::Punct { char: sep, spacing: tt::Spacing::Alone });
token_trees.push(punct.into());
has_sep = true;
if let Some(ref sep) = repeat.separator {
match sep {
crate::Separator::Ident(ident) => {
has_seps = 1;
token_trees.push(tt::Leaf::from(ident.clone()).into());
}
crate::Separator::Literal(lit) => {
has_seps = 1;
token_trees.push(tt::Leaf::from(lit.clone()).into());
}

crate::Separator::Puncts(puncts) => {
has_seps = puncts.len();
for punct in puncts {
token_trees.push(tt::Leaf::from(*punct).into());
}
}
}
}
}
nesting.pop().unwrap();

// Dirty hack for remove the last sep
// if it is a "," undo the push
if has_sep && repeat.separator.unwrap() == ',' {
for _ in 0..has_seps {
token_trees.pop();
}

Expand Down
Loading