Auto merge of #72287 - Aaron1011:feature/min-token-collect, r=petrochenkov

Store tokens inside `ast::Expr`

This is a smaller version of #70091.

We now store captured tokens inside `ast::Expr`, which allows us to avoid some reparsing in `nt_to_tokenstream`. To try to mitigate the performance impact, we only collect tokens when we've seen an outer attribute.

This makes progress towards solving #43081. There are still many things left to do:

* Collect tokens for other AST items.
* Come up with a way to handle inner attributes (we need to be collecting tokens by the time we encounter them)
* Avoid re-parsing when a `#[cfg]` attr is used.

However, this is enough to fix spans for a simple example, which I've included as a test case.

Author: bors

src/librustc_ast/ast.rs

@@ -1006,11 +1006,12 @@ pub struct Expr {
     pub kind: ExprKind,
     pub span: Span,
     pub attrs: AttrVec,
+    pub tokens: Option<TokenStream>,
 }
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Expr, 96);
+rustc_data_structures::static_assert_size!(Expr, 104);
 
 impl Expr {
     /// Returns `true` if this expression would be valid somewhere that expects a value;

src/librustc_ast/mut_visit.rs

@@ -1095,7 +1095,10 @@ pub fn noop_visit_anon_const<T: MutVisitor>(AnonConst { id, value }: &mut AnonCo
     vis.visit_expr(value);
 }
 
-pub fn noop_visit_expr<T: MutVisitor>(Expr { kind, id, span, attrs }: &mut Expr, vis: &mut T) {
+pub fn noop_visit_expr<T: MutVisitor>(
+    Expr { kind, id, span, attrs, tokens: _ }: &mut Expr,
+    vis: &mut T,
+) {
     match kind {
         ExprKind::Box(expr) => vis.visit_expr(expr),
         ExprKind::Array(exprs) => visit_exprs(exprs, vis),

src/librustc_ast_lowering/lib.rs

@@ -1126,6 +1126,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                 kind: ExprKind::Path(qself.clone(), path.clone()),
                                 span: ty.span,
                                 attrs: AttrVec::new(),
+                                tokens: None,
                             };
 
                             let ct = self.with_new_scopes(|this| hir::AnonConst {

src/librustc_builtin_macros/asm.rs

@@ -519,6 +519,7 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
         kind: ast::ExprKind::InlineAsm(inline_asm),
         span: sp,
         attrs: ast::AttrVec::new(),
+        tokens: None,
     })
 }
 

src/librustc_builtin_macros/concat_idents.rs

@@ -52,6 +52,7 @@ pub fn expand_concat_idents<'cx>(
                 kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)),
                 span: self.ident.span,
                 attrs: ast::AttrVec::new(),
+                tokens: None,
             }))
         }
 

src/librustc_builtin_macros/llvm_asm.rs

@@ -61,6 +61,7 @@ pub fn expand_llvm_asm<'cx>(
         kind: ast::ExprKind::LlvmInlineAsm(P(inline_asm)),
         span: cx.with_def_site_ctxt(sp),
         attrs: ast::AttrVec::new(),
+        tokens: None,
     }))
 }
 

src/librustc_expand/base.rs

@@ -594,6 +594,7 @@ impl DummyResult {
             kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) },
             span: sp,
             attrs: ast::AttrVec::new(),
+            tokens: None,
         })
     }
 

src/librustc_expand/build.rs

@@ -70,7 +70,13 @@ impl<'a> ExtCtxt<'a> {
     pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst {
         ast::AnonConst {
             id: ast::DUMMY_NODE_ID,
-            value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }),
+            value: P(ast::Expr {
+                id: ast::DUMMY_NODE_ID,
+                kind,
+                span,
+                attrs: AttrVec::new(),
+                tokens: None,
+            }),
         }
     }
 
@@ -205,7 +211,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P<ast::Expr> {
-        P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() })
+        P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new(), tokens: None })
     }
 
     pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> {

src/librustc_expand/placeholders.rs

@@ -34,6 +34,7 @@ pub fn placeholder(
             span,
             attrs: ast::AttrVec::new(),
             kind: ast::ExprKind::MacCall(mac_placeholder()),
+            tokens: None,
         })
     };
     let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span });

src/librustc_interface/util.rs

@@ -713,6 +713,7 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
                 kind: ast::ExprKind::Block(P(b), None),
                 span: rustc_span::DUMMY_SP,
                 attrs: AttrVec::new(),
+                tokens: None,
             });
 
             ast::Stmt {
@@ -728,6 +729,7 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
             id: self.resolver.next_node_id(),
             span: rustc_span::DUMMY_SP,
             attrs: AttrVec::new(),
+            tokens: None,
         });
 
         let loop_stmt = ast::Stmt {