Show invisible delimeters (within comments) when pretty printing.

compiler/rustc_ast/src/token.rs

@@ -50,11 +50,12 @@ pub enum Delimiter {
     Brace,
     /// `[ ... ]`
     Bracket,
-    /// `Ø ... Ø`
+    /// `/*«*/ ... /*»*/`
     /// An invisible delimiter, that may, for example, appear around tokens coming from a
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
-    /// Invisible delimiters might not survive roundtrip of a token stream through a string.
+    /// Invisible delimiters are not directly writable in normal Rust code except as comments.
+    /// Therefore, they might not survive a roundtrip of a token stream through a string.
     Invisible,
 }
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -582,15 +582,29 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
                     self.nbsp();
                 }
                 self.word("{");
-                if !tts.is_empty() {
+                let empty = tts.is_empty();
+                if !empty {
                     self.space();
                 }
                 self.ibox(0);
                 self.print_tts(tts, convert_dollar_crate);
                 self.end();
-                let empty = tts.is_empty();
                 self.bclose(span, empty);
             }
+            Some(Delimiter::Invisible) => {
+                self.word("/*«*/");
+                let empty = tts.is_empty();
+                if !empty {
+                    self.space();
+                }
+                self.ibox(0);
+                self.print_tts(tts, convert_dollar_crate);
+                self.end();
+                if !empty {
+                    self.space();
+                }
+                self.word("/*»*/");
+            }
             Some(delim) => {
                 let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
                 self.word(token_str);
@@ -764,9 +778,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::CloseDelim(Delimiter::Bracket) => "]".into(),
             token::OpenDelim(Delimiter::Brace) => "{".into(),
             token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
-                "".into()
-            }
+            token::OpenDelim(Delimiter::Invisible) => "/*«*/".into(),
+            token::CloseDelim(Delimiter::Invisible) => "/*»*/".into(),
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

library/proc_macro/src/lib.rs

@@ -703,11 +703,12 @@ pub enum Delimiter {
     /// `[ ... ]`
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     Bracket,
-    /// `Ø ... Ø`
+    /// `/*«*/ ... /*»*/`
     /// An invisible delimiter, that may, for example, appear around tokens coming from a
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
-    /// Invisible delimiters might not survive roundtrip of a token stream through a string.
+    /// Invisible delimiters are not directly writable in normal Rust code except as comments.
+    /// Therefore, they might not survive a roundtrip of a token stream through a string.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     None,
 }

src/test/ui/proc-macro/auxiliary/expand-expr.rs

@@ -12,6 +12,15 @@ use std::str::FromStr;
 
 #[proc_macro]
 pub fn expand_expr_is(input: TokenStream) -> TokenStream {
+    expand_expr_is_inner(input, false)
+}
+
+#[proc_macro]
+pub fn expand_expr_is_trim(input: TokenStream) -> TokenStream {
+    expand_expr_is_inner(input, true)
+}
+
+fn expand_expr_is_inner(input: TokenStream, trim_invisible: bool) -> TokenStream {
     let mut iter = input.into_iter();
     let mut expected_tts = Vec::new();
     loop {
@@ -22,14 +31,18 @@ pub fn expand_expr_is(input: TokenStream) -> TokenStream {
         }
     }
 
-    let expected = expected_tts.into_iter().collect::<TokenStream>();
-    let expanded = iter.collect::<TokenStream>().expand_expr().expect("expand_expr failed");
-    assert!(
-        expected.to_string() == expanded.to_string(),
-        "assert failed\nexpected: `{}`\nexpanded: `{}`",
-        expected.to_string(),
-        expanded.to_string()
-    );
+    // If requested, trim the "invisible" delimiters at the start and end.
+    let expected = expected_tts.into_iter().collect::<TokenStream>().to_string();
+    let expected = if trim_invisible {
+        let len1 = "/*«*/ ".len();
+        let len2 = " /*»*/".len();
+        &expected[len1..expected.len() - len2]
+    } else {
+        &expected[..]
+    };
+    let expanded = iter.collect::<TokenStream>().expand_expr().unwrap().to_string();
+
+    assert_eq!(expected, expanded);
 
     TokenStream::new()
 }

src/test/ui/proc-macro/capture-macro-rules-invoke.stdout

@@ -1,4 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): self
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ self /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,
@@ -13,8 +14,10 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
 ]
 PRINT-BANG INPUT (DISPLAY): 1 + 1, { "a" }, let a = 1;, String, my_name, 'a, my_val = 30,
 std::option::Option, pub(in some::path) , [a b c], -30
-PRINT-BANG RE-COLLECTED (DISPLAY): 1 + 1, { "a" }, let a = 1, String, my_name, 'a, my_val = 30,
-std :: option :: Option, pub(in some :: path), [a b c], - 30
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 1 + 1 /*»*/, /*«*/ { "a" } /*»*/, /*«*/ let a = 1 /*»*/, /*«*/
+String /*»*/, my_name, /*«*/ 'a /*»*/, /*«*/ my_val = 30 /*»*/, /*«*/
+std :: option :: Option /*»*/, /*«*/ pub(in some :: path) /*»*/, [a b c],
+/*«*/ - 30 /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,
@@ -295,6 +298,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): (a, b)
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ (a, b) /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/capture-unglued-token.stdout

@@ -1,5 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): Vec<u8>
-PRINT-BANG RE-COLLECTED (DISPLAY): Vec < u8 >
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ Vec < u8 > /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/expand-expr.rs

@@ -2,9 +2,9 @@
 
 extern crate expand_expr;
 
-use expand_expr::{
-    check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is, recursive_expand,
-};
+use expand_expr::{check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is};
+use expand_expr::{expand_expr_is_trim, recursive_expand};
+
 
 // Check builtin macros can be expanded.
 
@@ -47,21 +47,21 @@ macro_rules! echo_expr {
 
 macro_rules! simple_lit {
     ($l:literal) => {
-        expand_expr_is!($l, $l);
-        expand_expr_is!($l, echo_lit!($l));
-        expand_expr_is!($l, echo_expr!($l));
-        expand_expr_is!($l, echo_tts!($l));
-        expand_expr_is!($l, echo_pm!($l));
+        expand_expr_is_trim!($l, $l);
+        expand_expr_is_trim!($l, echo_lit!($l));
+        expand_expr_is_trim!($l, echo_expr!($l));
+        expand_expr_is_trim!($l, echo_tts!($l));
+        expand_expr_is_trim!($l, echo_pm!($l));
         const _: () = {
             macro_rules! mac {
                 () => {
                     $l
                 };
             }
-            expand_expr_is!($l, mac!());
-            expand_expr_is!($l, echo_expr!(mac!()));
-            expand_expr_is!($l, echo_tts!(mac!()));
-            expand_expr_is!($l, echo_pm!(mac!()));
+            expand_expr_is_trim!($l, mac!());
+            expand_expr_is_trim!($l, echo_expr!(mac!()));
+            expand_expr_is_trim!($l, echo_tts!(mac!()));
+            expand_expr_is_trim!($l, echo_pm!(mac!()));
         };
     };
 }

src/test/ui/proc-macro/expr-stmt-nonterminal-tokens.stdout

@@ -1,5 +1,6 @@
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = #[allow(warnings)] 0 ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = #[allow(warnings)] #[allow(warnings)] 0 ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E
+{ V = { let _ = /*«*/ #[allow(warnings)] #[allow(warnings)] 0 /*»*/ ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -123,7 +124,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { 0; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { 0 } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ 0 /*»*/ } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -203,6 +204,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { {} } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ {} /*»*/ } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -281,7 +283,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { PATH; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { PATH } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ PATH /*»*/ } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -359,7 +361,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { 0 + 1; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { 0 + 1 } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ 0 + 1 /*»*/ } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",
@@ -450,7 +452,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): enum E { V = { let _ = { PATH + 1; } ; 0 }, }
-PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { PATH + 1 } ; 0 }, }
+PRINT-DERIVE DEEP-RE-COLLECTED (DISPLAY): enum E { V = { let _ = { /*«*/ PATH + 1 /*»*/ } ; 0 }, }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "enum",

src/test/ui/proc-macro/issue-75734-pp-paren.stdout

@@ -96,6 +96,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): 1 + 1 * 2
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 1 + 1 /*»*/ * 2
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/issue-78675-captured-inner-attrs.stdout

@@ -1,7 +1,7 @@
 PRINT-BANG INPUT (DISPLAY): foo! { #[fake_attr] mod bar {
     #![doc = r" Foo"]
 } }
-PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): foo! { #[fake_attr] mod bar { #! [doc = r" Foo"] } }
+PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): foo! { #[fake_attr] /*«*/ mod bar { #! [doc = r" Foo"] } /*»*/ }
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Ident {
         ident: "foo",

src/test/ui/proc-macro/issue-80760-empty-stmt.stdout

@@ -1,4 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): ;
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ ; /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/nested-nonterminal-tokens.stdout

@@ -1,4 +1,6 @@
 PRINT-BANG INPUT (DISPLAY): 0 + 1 + 2 + 3
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ 0 + 1 + 2 /*»*/ + 3
+PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): /*«*/ /*«*/ /*«*/ 0 /*»*/ + 1 /*»*/ + 2 /*»*/ + 3
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/nodelim-groups.stdout

@@ -1,4 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): "hi" 1 + (25) + 1 (1 + 1)
+PRINT-BANG RE-COLLECTED (DISPLAY): "hi" /*«*/ 1 + (25) + 1 /*»*/ (1 + 1)
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Literal {
         kind: Str,
@@ -71,6 +72,9 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): "hi" "hello".len() + "world".len() (1 + 1)
+PRINT-BANG RE-COLLECTED (DISPLAY): "hi" /*«*/ "hello".len() + "world".len() /*»*/ (1 + 1)
+PRINT-BANG DEEP-RE-COLLECTED (DISPLAY): "hi" /*«*/ /*«*/ "hello".len() /*»*/ + /*«*/ "world".len() /*»*/ /*»*/
+(1 + 1)
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Literal {
         kind: Str,

src/test/ui/proc-macro/nonterminal-expansion.stdout

@@ -1,5 +1,5 @@
 PRINT-ATTR_ARGS INPUT (DISPLAY): a, line!(), b
-PRINT-ATTR_ARGS RE-COLLECTED (DISPLAY): a, line! (), b
+PRINT-ATTR_ARGS RE-COLLECTED (DISPLAY): a, /*«*/ line! () /*»*/, b
 PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [
     Ident {
         ident: "a",

src/test/ui/proc-macro/nonterminal-token-hygiene.stdout

@@ -1,5 +1,5 @@
 PRINT-BANG INPUT (DISPLAY): struct S;
-PRINT-BANG RE-COLLECTED (DISPLAY): struct S ;
+PRINT-BANG RE-COLLECTED (DISPLAY): /*«*/ struct S ; /*»*/
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/parent-source-spans.rs

@@ -8,16 +8,16 @@ use parent_source_spans::parent_source_spans;
 
 macro one($a:expr, $b:expr) {
     two!($a, $b);
-    //~^ ERROR first parent: "hello"
-    //~| ERROR second parent: "world"
+    //~^ ERROR first parent: /*«*/ "hello" /*»*/
+    //~| ERROR second parent: /*«*/ "world" /*»*/
 }
 
 macro two($a:expr, $b:expr) {
     three!($a, $b);
-    //~^ ERROR first final: "hello"
-    //~| ERROR second final: "world"
-    //~| ERROR first final: "yay"
-    //~| ERROR second final: "rust"
+    //~^ ERROR first final: /*«*/ "hello" /*»*/
+    //~| ERROR second final: /*«*/ "world" /*»*/
+    //~| ERROR first final: /*«*/ "yay" /*»*/
+    //~| ERROR second final: /*«*/ "rust" /*»*/
 }
 
 // forwarding tokens directly doesn't create a new source chain
@@ -34,16 +34,16 @@ macro four($($tokens:tt)*) {
 
 fn main() {
     one!("hello", "world");
-    //~^ ERROR first grandparent: "hello"
-    //~| ERROR second grandparent: "world"
-    //~| ERROR first source: "hello"
-    //~| ERROR second source: "world"
+    //~^ ERROR first grandparent: /*«*/ "hello" /*»*/
+    //~| ERROR second grandparent: /*«*/ "world" /*»*/
+    //~| ERROR first source: /*«*/ "hello" /*»*/
+    //~| ERROR second source: /*«*/ "world" /*»*/
 
     two!("yay", "rust");
-    //~^ ERROR first parent: "yay"
-    //~| ERROR second parent: "rust"
-    //~| ERROR first source: "yay"
-    //~| ERROR second source: "rust"
+    //~^ ERROR first parent: /*«*/ "yay" /*»*/
+    //~| ERROR second parent: /*«*/ "rust" /*»*/
+    //~| ERROR first source: /*«*/ "yay" /*»*/
+    //~| ERROR second source: /*«*/ "rust" /*»*/
 
     three!("hip", "hop");
     //~^ ERROR first final: "hip"