[WIP] Treat crates specially when expanding

src/libsyntax/ast.rs

@@ -454,6 +454,7 @@ pub struct Crate {
     pub module: Mod,
     pub attrs: Vec<Attribute>,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 /// A spanned compile-time attribute list item.

src/libsyntax/attr.rs

@@ -1383,6 +1383,6 @@ macro_rules! derive_has_attrs {
 }
 
 derive_has_attrs! {
-    Item, Expr, Local, ast::ForeignItem, ast::StructField, ast::ImplItem, ast::TraitItem, ast::Arm,
-    ast::Field, ast::FieldPat, ast::Variant_
+    Item, Expr, Local, ast::ForeignItem, ast::StructField, ast::ImplItem, ast::TraitItem,
+    ast::Arm, ast::Field, ast::FieldPat, ast::Variant_, ast::Crate
 }

src/libsyntax/ext/base.rs

@@ -35,6 +35,7 @@ use tokenstream::{self, TokenStream};
 
 #[derive(Debug,Clone)]
 pub enum Annotatable {
+    Crate(P<ast::Crate>),
     Item(P<ast::Item>),
     TraitItem(P<ast::TraitItem>),
     ImplItem(P<ast::ImplItem>),
@@ -46,6 +47,7 @@ pub enum Annotatable {
 impl HasAttrs for Annotatable {
     fn attrs(&self) -> &[Attribute] {
         match *self {
+            Annotatable::Crate(ref krate) => &krate.attrs,
             Annotatable::Item(ref item) => &item.attrs,
             Annotatable::TraitItem(ref trait_item) => &trait_item.attrs,
             Annotatable::ImplItem(ref impl_item) => &impl_item.attrs,
@@ -57,6 +59,7 @@ impl HasAttrs for Annotatable {
 
     fn map_attrs<F: FnOnce(Vec<Attribute>) -> Vec<Attribute>>(self, f: F) -> Self {
         match self {
+            Annotatable::Crate(krate) => Annotatable::Crate(krate.map_attrs(f)),
             Annotatable::Item(item) => Annotatable::Item(item.map_attrs(f)),
             Annotatable::TraitItem(trait_item) => Annotatable::TraitItem(trait_item.map_attrs(f)),
             Annotatable::ImplItem(impl_item) => Annotatable::ImplItem(impl_item.map_attrs(f)),
@@ -71,6 +74,7 @@ impl HasAttrs for Annotatable {
 impl Annotatable {
     pub fn span(&self) -> Span {
         match *self {
+            Annotatable::Crate(ref krate) => krate.span,
             Annotatable::Item(ref item) => item.span,
             Annotatable::TraitItem(ref trait_item) => trait_item.span,
             Annotatable::ImplItem(ref impl_item) => impl_item.span,
@@ -87,6 +91,13 @@ impl Annotatable {
         }
     }
 
+    pub fn expect_crate(self) -> ast::Crate {
+        match self {
+            Annotatable::Crate(c) => c.into_inner(),
+            _ => panic!("expected crate"),
+        }
+    }
+
     pub fn map_item_or<F, G>(self, mut f: F, mut or: G) -> Annotatable
         where F: FnMut(P<ast::Item>) -> P<ast::Item>,
               G: FnMut(Annotatable) -> Annotatable
@@ -118,6 +129,20 @@ impl Annotatable {
         }
     }
 
+    pub fn expect_stmt(self) -> ast::Stmt {
+        match self {
+            Annotatable::Stmt(stmt) => stmt.into_inner(),
+            _ => panic!("expected statement"),
+        }
+    }
+
+    pub fn expect_expr(self) -> P<ast::Expr> {
+        match self {
+            Annotatable::Expr(expr) => expr,
+            _ => panic!("expected expression"),
+        }
+    }
+
     pub fn derive_allowed(&self) -> bool {
         match *self {
             Annotatable::Item(ref item) => match item.node {
@@ -310,6 +335,11 @@ macro_rules! make_stmts_default {
 /// The result of a macro expansion. The return values of the various
 /// methods are spliced into the AST at the callsite of the macro.
 pub trait MacResult {
+    /// Create a whole crate.
+    fn make_crate(self: Box<Self>) -> Option<P<ast::Crate>> {
+        None
+    }
+
     /// Create an expression.
     fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
         None
@@ -375,6 +405,7 @@ macro_rules! make_MacEager {
 }
 
 make_MacEager! {
+    krate: P<ast::Crate>,
     expr: P<ast::Expr>,
     pat: P<ast::Pat>,
     items: SmallVector<P<ast::Item>>,
@@ -386,6 +417,10 @@ make_MacEager! {
 }
 
 impl MacResult for MacEager {
+    fn make_crate(self: Box<Self>) -> Option<P<ast::Crate>> {
+        self.krate
+    }
+
     fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
         self.expr
     }
@@ -489,6 +524,20 @@ impl DummyResult {
 }
 
 impl MacResult for DummyResult {
+    fn make_crate(self: Box<DummyResult>) -> Option<P<ast::Crate>> {
+        Some(P(
+            ast::Crate{
+                attrs: vec![],
+                module: ast::Mod {
+                    inner: self.span,
+                    items: vec![],
+                },
+                span: self.span,
+                tokens: None,
+            }
+        ))
+    }
+
     fn make_expr(self: Box<DummyResult>) -> Option<P<ast::Expr>> {
         Some(DummyResult::raw_expr(self.span))
     }

src/libsyntax/ext/expand.rs

@@ -45,20 +45,27 @@ macro_rules! expansions {
             $(.$fold:ident)*  $(lift .$fold_elt:ident)*,
             $(.$visit:ident)*  $(lift .$visit_elt:ident)*;)*) => {
         #[derive(Copy, Clone, PartialEq, Eq)]
-        pub enum ExpansionKind { OptExpr, $( $kind, )*  }
-        pub enum Expansion { OptExpr(Option<P<ast::Expr>>), $( $kind($ty), )* }
+        pub enum ExpansionKind { OptExpr, Crate, $( $kind, )*  }
+        pub enum Expansion {
+            OptExpr(Option<P<ast::Expr>>),
+            // we don't box `ast::Crate` in most places but we do it here to keep `Expansion` small
+            Crate(P<ast::Crate>),
+            $( $kind($ty), )*
+        }
 
         impl ExpansionKind {
             pub fn name(self) -> &'static str {
                 match self {
                     ExpansionKind::OptExpr => "expression",
+                    ExpansionKind::Crate => "crate",
                     $( ExpansionKind::$kind => $kind_name, )*
                 }
             }
 
             fn make_from<'a>(self, result: Box<MacResult + 'a>) -> Option<Expansion> {
                 match self {
                     ExpansionKind::OptExpr => result.make_expr().map(Some).map(Expansion::OptExpr),
+                    ExpansionKind::Crate => result.make_crate().map(Expansion::Crate),
                     $( ExpansionKind::$kind => result.$make().map(Expansion::$kind), )*
                 }
             }
@@ -71,6 +78,14 @@ macro_rules! expansions {
                     _ => panic!("Expansion::make_* called on the wrong kind of expansion"),
                 }
             }
+
+            pub fn make_crate(self) -> ast::Crate {
+                match self {
+                    Expansion::Crate(krate) => krate.into_inner(),
+                    _ => panic!("Expansion::make_* called on the wrong kind of expansion"),
+                }
+            }
+
             $( pub fn $make(self) -> $ty {
                 match self {
                     Expansion::$kind(ast) => ast,
@@ -82,6 +97,7 @@ macro_rules! expansions {
                 use self::Expansion::*;
                 match self {
                     OptExpr(expr) => OptExpr(expr.and_then(|expr| folder.fold_opt_expr(expr))),
+                    Crate(krate) => Crate(krate.map(|k| folder.fold_crate(k))),
                     $($( $kind(ast) => $kind(folder.$fold(ast)), )*)*
                     $($( $kind(ast) => {
                         $kind(ast.into_iter().flat_map(|ast| folder.$fold_elt(ast)).collect())
@@ -91,6 +107,7 @@ macro_rules! expansions {
 
             pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
                 match *self {
+                    Expansion::Crate(ref krate) => visitor.visit_crate(krate),
                     Expansion::OptExpr(Some(ref expr)) => visitor.visit_expr(expr),
                     Expansion::OptExpr(None) => {}
                     $($( Expansion::$kind(ref ast) => visitor.$visit(ast), )*)*
@@ -105,6 +122,11 @@ macro_rules! expansions {
             fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
                 self.expand(Expansion::OptExpr(Some(expr))).make_opt_expr()
             }
+
+            fn fold_crate(&mut self, krate: ast::Crate) -> ast::Crate {
+                self.expand(Expansion::Crate(P(krate))).make_crate()
+            }
+
             $($(fn $fold(&mut self, node: $ty) -> $ty {
                 self.expand(Expansion::$kind(node)).$make()
             })*)*
@@ -143,8 +165,12 @@ impl ExpansionKind {
     }
 
     fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I) -> Expansion {
-        let items = items.into_iter();
+        let mut items = items.into_iter();
         match self {
+            ExpansionKind::Crate => Expansion::Crate(
+                // we only box `ast::Crate` for `Expansion`
+                P(items.next().expect("expected exactly one crate").expect_crate()),
+            ),
             ExpansionKind::Items =>
                 Expansion::Items(items.map(Annotatable::expect_item).collect()),
             ExpansionKind::ImplItems =>
@@ -153,7 +179,14 @@ impl ExpansionKind {
                 Expansion::TraitItems(items.map(Annotatable::expect_trait_item).collect()),
             ExpansionKind::ForeignItems =>
                 Expansion::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()),
-            _ => unreachable!(),
+            ExpansionKind::Stmts => Expansion::Stmts(items.map(Annotatable::expect_stmt).collect()),
+            ExpansionKind::Expr => Expansion::Expr(
+                items.next().expect("expected exactly one expression").expect_expr()
+            ),
+            ExpansionKind::OptExpr =>
+                Expansion::OptExpr(items.next().map(Annotatable::expect_expr)),
+            ExpansionKind::Pat | ExpansionKind::Ty =>
+                panic!("patterns and types aren't annotatable"),
         }
     }
 }
@@ -235,33 +268,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         self.cx.current_expansion.module = Rc::new(module);
         self.cx.current_expansion.crate_span = Some(krate.span);
 
-        let orig_mod_span = krate.module.inner;
-
-        let krate_item = Expansion::Items(SmallVector::one(P(ast::Item {
-            attrs: krate.attrs,
-            span: krate.span,
-            node: ast::ItemKind::Mod(krate.module),
-            ident: keywords::Invalid.ident(),
-            id: ast::DUMMY_NODE_ID,
-            vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public),
-            tokens: None,
-        })));
-
-        match self.expand(krate_item).make_items().pop().map(P::into_inner) {
-            Some(ast::Item { attrs, node: ast::ItemKind::Mod(module), .. }) => {
-                krate.attrs = attrs;
-                krate.module = module;
-            },
-            None => {
-                // Resolution failed so we return an empty expansion
-                krate.attrs = vec![];
-                krate.module = ast::Mod {
-                    inner: orig_mod_span,
-                    items: vec![],
-                };
-            },
-            _ => unreachable!(),
-        };
+        // we only box `ast::Crate` for `Expansion` to keep it small
+        let krate_item = Expansion::Crate(P(krate));
+        krate = self.expand(krate_item).make_crate();
+
         self.cx.trace_macros_diag();
         krate
     }
@@ -430,6 +440,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         // Since the item itself has already been configured by the InvocationCollector,
         // we know that fold result vector will contain exactly one element
         match item {
+            Annotatable::Crate(krate) => {
+                Annotatable::Crate(krate.map(|k| cfg.fold_crate(k)))
+            }
             Annotatable::Item(item) => {
                 Annotatable::Item(cfg.fold_item(item).pop().unwrap())
             }
@@ -515,6 +528,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             }
             AttrProcMacro(ref mac) => {
                 let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
+                    Annotatable::Crate(krate) => token::NtCrate(krate),
                     Annotatable::Item(item) => token::NtItem(item),
                     Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
                     Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
@@ -782,6 +796,9 @@ impl<'a> Parser<'a> {
     pub fn parse_expansion(&mut self, kind: ExpansionKind, macro_legacy_warnings: bool)
                            -> PResult<'a, Expansion> {
         Ok(match kind {
+            ExpansionKind::Crate => {
+                Expansion::Crate(P(self.parse_crate_mod()?))
+            }
             ExpansionKind::Items => {
                 let mut items = SmallVector::new();
                 while let Some(item) = self.parse_item()? {

src/libsyntax/ext/placeholders.rs

@@ -42,6 +42,14 @@ pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion {
     });
 
     match kind {
+        ExpansionKind::Crate => Expansion::Crate(P(ast::Crate {
+            attrs, module: ast::Mod {
+                inner: span,
+                items: vec![],
+            },
+            span,
+            tokens: None,
+        })),
         ExpansionKind::Expr => Expansion::Expr(expr_placeholder()),
         ExpansionKind::OptExpr => Expansion::OptExpr(Some(expr_placeholder())),
         ExpansionKind::Items => Expansion::Items(SmallVector::one(P(ast::Item {

src/libsyntax/fold.rs

@@ -617,6 +617,7 @@ pub fn noop_fold_token<T: Folder>(t: token::Token, fld: &mut T) -> token::Token
 pub fn noop_fold_interpolated<T: Folder>(nt: token::Nonterminal, fld: &mut T)
                                          -> token::Nonterminal {
     match nt {
+        token::NtCrate(krate) => token::NtCrate(krate.map(|k| fld.fold_crate(k))),
         token::NtItem(item) =>
             token::NtItem(fld.fold_item(item)
                           // this is probably okay, because the only folds likely
@@ -1017,7 +1018,7 @@ pub fn noop_fold_mod<T: Folder>(Mod {inner, items}: Mod, folder: &mut T) -> Mod
     }
 }
 
-pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, span}: Crate,
+pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, span, tokens}: Crate,
                                   folder: &mut T) -> Crate {
     let mut items = folder.fold_item(P(ast::Item {
         ident: keywords::Invalid.ident(),
@@ -1026,30 +1027,31 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, span}: Crate,
         vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Public),
         span,
         node: ast::ItemKind::Mod(module),
-        tokens: None,
+        tokens,
     })).into_iter();
 
-    let (module, attrs, span) = match items.next() {
+    let (module, attrs, span, tokens) = match items.next() {
         Some(item) => {
             assert!(items.next().is_none(),
                     "a crate cannot expand to more than one item");
-            item.and_then(|ast::Item { attrs, span, node, .. }| {
+            item.and_then(|ast::Item { attrs, span, node, tokens, .. }| {
                 match node {
-                    ast::ItemKind::Mod(m) => (m, attrs, span),
+                    ast::ItemKind::Mod(m) => (m, attrs, span, tokens),
                     _ => panic!("fold converted a module to not a module"),
                 }
             })
         }
         None => (ast::Mod {
             inner: span,
             items: vec![],
-        }, vec![], span)
+        }, vec![], span, None)
     };
 
     Crate {
         module,
         attrs,
         span,
+        tokens,
     }
 }
 

src/libsyntax/parse/parser.rs

@@ -7047,6 +7047,7 @@ impl<'a> Parser<'a> {
             attrs: self.parse_inner_attributes()?,
             module: self.parse_mod_items(&token::Eof, lo)?,
             span: lo.to(self.span),
+            tokens: None,
         })
     }