Skip to content

Commit

Permalink
Rollup merge of rust-lang#56964 - nnethercote:TokenStream-IsJoint, r=…
Browse files Browse the repository at this point in the history 
…petrochenkov

Remove `TokenStream::JointTree`.

This is done by adding a new `IsJoint` field to `TokenStream::Tree`,
which simplifies a lot of `match` statements. And likewise for
`CursorKind`.

The commit also adds a new method `TokenTree:stream()` which can replace
a choice between `.into()` and `.joint()`.
  • Loading branch information
@pietroalbini
pietroalbini committed Dec 21, 2018
2 parents 652e138 + e7c5146 commit c2444d4
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 60 deletions.
5 changes: 2 additions & 3 deletions src/libsyntax/parse/lexer/tokentrees.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
use print::pprust::token_to_string;
use parse::lexer::StringReader;
use parse::{token, PResult};
use tokenstream::{DelimSpan, TokenStream, TokenTree};
use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree};

impl<'a> StringReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
Expand Down Expand Up @@ -186,8 +186,7 @@ impl<'a> StringReader<'a> {
let raw = self.span_src_raw;
self.real_token();
let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);

Ok(if is_joint { tt.joint() } else { tt.into() })
Ok(TokenStream::Tree(tt, if is_joint { Joint } else { NonJoint }))
}
}
}
Expand Down
93 changes: 42 additions & 51 deletions src/libsyntax/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ impl TokenTree {
}

pub fn joint(self) -> TokenStream {
TokenStream::JointTree(self)
TokenStream::Tree(self, Joint)
}

/// Returns the opening delimiter as a token tree.
Expand Down Expand Up @@ -156,15 +156,22 @@ impl TokenTree {
#[derive(Clone, Debug)]
pub enum TokenStream {
Empty,
Tree(TokenTree),
JointTree(TokenTree),
Tree(TokenTree, IsJoint),
Stream(Lrc<Vec<TokenStream>>),
}

// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::<TokenStream>() == 32);

#[derive(Clone, Copy, Debug, PartialEq)]
pub enum IsJoint {
Joint,
NonJoint
}

use self::IsJoint::*;

impl TokenStream {
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
/// separating the two arguments with a comma for diagnostic suggestions.
Expand All @@ -176,16 +183,16 @@ impl TokenStream {
while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) {
(TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) |
(_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => {
continue;
}
(TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp,
(TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(),
(TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint), _) |
(_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint))
=> continue,
(TokenStream::Tree(TokenTree::Token(sp, _), NonJoint), _) => *sp,
(TokenStream::Tree(TokenTree::Delimited(sp, ..), NonJoint), _) =>
sp.entire(),
_ => continue,
};
let sp = sp.shrink_to_hi();
let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma));
let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma), NonJoint);
suggestion = Some((pos, comma, sp));
}
}
Expand All @@ -204,7 +211,7 @@ impl TokenStream {

impl From<TokenTree> for TokenStream {
fn from(tt: TokenTree) -> TokenStream {
TokenStream::Tree(tt)
TokenStream::Tree(tt, NonJoint)
}
}

Expand Down Expand Up @@ -232,7 +239,7 @@ impl Extend<TokenStream> for TokenStream {
vec.reserve(iter.size_hint().0);
vec
}
TokenStream::Tree(_) | TokenStream::JointTree(_) => {
TokenStream::Tree(..) => {
let mut vec = Vec::new();
vec.reserve(1 + iter.size_hint().0);
vec.push(this);
Expand Down Expand Up @@ -369,8 +376,7 @@ impl TokenStream {
/// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
match self {
TokenStream::Tree(tree) => (tree, false),
TokenStream::JointTree(tree) => (tree, true),
TokenStream::Tree(tree, is_joint) => (tree, is_joint == Joint),
_ => unreachable!(),
}
}
Expand All @@ -381,8 +387,7 @@ impl TokenStream {
let mut i = 0;
while let Some(stream) = trees.next_as_stream() {
result.push(match stream {
TokenStream::Tree(tree) => f(i, tree).into(),
TokenStream::JointTree(tree) => f(i, tree).joint(),
TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(i, tree), is_joint),
_ => unreachable!()
});
i += 1;
Expand All @@ -395,27 +400,25 @@ impl TokenStream {
let mut result = Vec::new();
while let Some(stream) = trees.next_as_stream() {
result.push(match stream {
TokenStream::Tree(tree) => f(tree).into(),
TokenStream::JointTree(tree) => f(tree).joint(),
TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(tree), is_joint),
_ => unreachable!()
});
}
TokenStream::new(result)
}

fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
fn first_tree_and_joint(&self) -> Option<(TokenTree, IsJoint)> {
match self {
TokenStream::Empty => None,
TokenStream::Tree(ref tree) => Some((tree.clone(), false)),
TokenStream::JointTree(ref tree) => Some((tree.clone(), true)),
TokenStream::Tree(ref tree, is_joint) => Some((tree.clone(), *is_joint)),
TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
}
}

fn last_tree_if_joint(&self) -> Option<TokenTree> {
match self {
TokenStream::Empty | TokenStream::Tree(..) => None,
TokenStream::JointTree(ref tree) => Some(tree.clone()),
TokenStream::Empty | TokenStream::Tree(_, NonJoint) => None,
TokenStream::Tree(ref tree, Joint) => Some(tree.clone()),
TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
}
}
Expand All @@ -439,11 +442,7 @@ impl TokenStreamBuilder {
self.push_all_but_last_tree(&last_stream);
let glued_span = last_span.to(span);
let glued_tt = TokenTree::Token(glued_span, glued_tok);
let glued_tokenstream = if is_joint {
glued_tt.joint()
} else {
glued_tt.into()
};
let glued_tokenstream = TokenStream::Tree(glued_tt, is_joint);
self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream);
return
Expand Down Expand Up @@ -493,8 +492,7 @@ pub struct Cursor(CursorKind);
#[derive(Clone)]
enum CursorKind {
Empty,
Tree(TokenTree, bool /* consumed? */),
JointTree(TokenTree, bool /* consumed? */),
Tree(TokenTree, IsJoint, bool /* consumed? */),
Stream(StreamCursor),
}

Expand All @@ -516,9 +514,9 @@ impl StreamCursor {
self.index += 1;
let next = self.stream[self.index - 1].clone();
match next {
TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next),
TokenStream::Stream(stream) => self.insert(stream),
TokenStream::Empty => {}
TokenStream::Tree(..) => return Some(next),
TokenStream::Stream(stream) => self.insert(stream),
}
} else if let Some((stream, index)) = self.stack.pop() {
self.stream = stream;
Expand All @@ -540,7 +538,7 @@ impl Iterator for Cursor {

fn next(&mut self) -> Option<TokenTree> {
self.next_as_stream().map(|stream| match stream {
TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree,
TokenStream::Tree(tree, _) => tree,
_ => unreachable!()
})
}
Expand All @@ -550,18 +548,15 @@ impl Cursor {
fn new(stream: TokenStream) -> Self {
Cursor(match stream {
TokenStream::Empty => CursorKind::Empty,
TokenStream::Tree(tree) => CursorKind::Tree(tree, false),
TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false),
TokenStream::Tree(tree, is_joint) => CursorKind::Tree(tree, is_joint, false),
TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)),
})
}

pub fn next_as_stream(&mut self) -> Option<TokenStream> {
let (stream, consumed) = match self.0 {
CursorKind::Tree(ref tree, ref mut consumed @ false) =>
(tree.clone().into(), consumed),
CursorKind::JointTree(ref tree, ref mut consumed @ false) =>
(tree.clone().joint(), consumed),
CursorKind::Tree(ref tree, ref is_joint, ref mut consumed @ false) =>
(TokenStream::Tree(tree.clone(), *is_joint), consumed),
CursorKind::Stream(ref mut cursor) => return cursor.next_as_stream(),
_ => return None,
};
Expand All @@ -574,7 +569,7 @@ impl Cursor {
match self.0 {
_ if stream.is_empty() => return,
CursorKind::Empty => *self = stream.trees(),
CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => {
CursorKind::Tree(_, _, consumed) => {
*self = TokenStream::new(vec![self.original_stream(), stream]).trees();
if consumed {
self.next();
Expand All @@ -589,8 +584,8 @@ impl Cursor {
pub fn original_stream(&self) -> TokenStream {
match self.0 {
CursorKind::Empty => TokenStream::empty(),
CursorKind::Tree(ref tree, _) => tree.clone().into(),
CursorKind::JointTree(ref tree, _) => tree.clone().joint(),
CursorKind::Tree(ref tree, ref is_joint, _) =>
TokenStream::Tree(tree.clone(), *is_joint),
CursorKind::Stream(ref cursor) => TokenStream::Stream(
cursor.stack.get(0).cloned().map(|(stream, _)| stream)
.unwrap_or_else(|| cursor.stream.clone())
Expand All @@ -602,9 +597,8 @@ impl Cursor {
fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
for stream in streams {
n = match stream {
TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree)
if n == 0 => return Ok(tree.clone()),
TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1,
TokenStream::Tree(ref tree, _) if n == 0 => return Ok(tree.clone()),
TokenStream::Tree(..) => n - 1,
TokenStream::Stream(ref stream) => match look_ahead(stream, n) {
Ok(tree) => return Ok(tree),
Err(n) => n,
Expand All @@ -617,10 +611,8 @@ impl Cursor {

match self.0 {
CursorKind::Empty |
CursorKind::Tree(_, true) |
CursorKind::JointTree(_, true) => Err(n),
CursorKind::Tree(ref tree, false) |
CursorKind::JointTree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Tree(_, _, true) => Err(n),
CursorKind::Tree(ref tree, _, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Stream(ref cursor) => {
look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| {
for &(ref stream, index) in cursor.stack.iter().rev() {
Expand Down Expand Up @@ -653,8 +645,7 @@ impl From<TokenStream> for ThinTokenStream {
fn from(stream: TokenStream) -> ThinTokenStream {
ThinTokenStream(match stream {
TokenStream::Empty => None,
TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])),
TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])),
TokenStream::Tree(..) => Some(Lrc::new(vec![stream])),
TokenStream::Stream(stream) => Some(stream),
})
}
Expand Down
8 changes: 2 additions & 6 deletions src/libsyntax_ext/proc_macro_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use syntax::ast;
use syntax::ext::base::ExtCtxt;
use syntax::parse::lexer::comments;
use syntax::parse::{self, token, ParseSess};
use syntax::tokenstream::{self, DelimSpan, TokenStream};
use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream};
use syntax_pos::hygiene::{SyntaxContext, Transparency};
use syntax_pos::symbol::{keywords, Symbol};
use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
Expand Down Expand Up @@ -278,11 +278,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
};

let tree = tokenstream::TokenTree::Token(span, token);
if joint {
tree.joint()
} else {
tree.into()
}
TokenStream::Tree(tree, if joint { Joint } else { NonJoint })
}
}

Expand Down

0 comments on commit c2444d4

Please sign in to comment.