Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 0d4f9e5

Browse files
nnethercotennethercote
committed
Remove NtItem and NtStmt.
This involves replacing `nt_pretty_printing_compatibility_hack` with `stream_pretty_printing_compatibility_hack`. The handling of statements in `transcribe` is slightly different to other nonterminal kinds, due to the lack of `from_ast` implementation for empty statements. Notable test changes: - `tests/ui/macros/nonterminal-matching.rs` now passes. Removal of `Token::Interpolated` will remove the "captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens" restriction. - `tests/ui/proc-macro/expand-to-derive.rs`: the diff looks large but the only difference is the insertion of a single invisible-delimited group.
1 parent 78b3a8f commit 0d4f9e5

File tree

15 files changed

+160
-151
lines changed

15 files changed

+160
-151
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -231,8 +231,6 @@ impl HasTokens for Attribute {
231231
impl HasTokens for Nonterminal {
232232
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
233233
match self {
234-
Nonterminal::NtItem(item) => item.tokens(),
235-
Nonterminal::NtStmt(stmt) => stmt.tokens(),
236234
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
237235
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
238236
Nonterminal::NtPath(path) => path.tokens(),
@@ -241,8 +239,6 @@ impl HasTokens for Nonterminal {
241239
}
242240
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
243241
match self {
244-
Nonterminal::NtItem(item) => item.tokens_mut(),
245-
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
246242
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
247243
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
248244
Nonterminal::NtPath(path) => path.tokens_mut(),

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -819,19 +819,7 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
819819
// multiple items there....
820820
fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
821821
match nt {
822-
token::NtItem(item) => visit_clobber(item, |item| {
823-
// This is probably okay, because the only visitors likely to
824-
// peek inside interpolated nodes will be renamings/markings,
825-
// which map single items to single items.
826-
vis.flat_map_item(item).expect_one("expected visitor to produce exactly one item")
827-
}),
828822
token::NtBlock(block) => vis.visit_block(block),
829-
token::NtStmt(stmt) => visit_clobber(stmt, |stmt| {
830-
// See reasoning above.
831-
stmt.map(|stmt| {
832-
vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item")
833-
})
834-
}),
835823
token::NtExpr(expr) => vis.visit_expr(expr),
836824
token::NtLiteral(expr) => vis.visit_expr(expr),
837825
token::NtMeta(item) => {

‎compiler/rustc_ast/src/token.rs

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -939,9 +939,7 @@ impl PartialEq<TokenKind> for Token {
939939
#[derive(Clone, Encodable, Decodable)]
940940
/// For interpolation during macro expansion.
941941
pub enum Nonterminal {
942-
NtItem(P<ast::Item>),
943942
NtBlock(P<ast::Block>),
944-
NtStmt(P<ast::Stmt>),
945943
NtExpr(P<ast::Expr>),
946944
NtLiteral(P<ast::Expr>),
947945
/// Stuff inside brackets for attributes
@@ -1043,9 +1041,7 @@ impl fmt::Display for NonterminalKind {
10431041
impl Nonterminal {
10441042
pub fn use_span(&self) -> Span {
10451043
match self {
1046-
NtItem(item) => item.span,
10471044
NtBlock(block) => block.span,
1048-
NtStmt(stmt) => stmt.span,
10491045
NtExpr(expr) | NtLiteral(expr) => expr.span,
10501046
NtMeta(attr_item) => attr_item.span(),
10511047
NtPath(path) => path.span,
@@ -1054,9 +1050,7 @@ impl Nonterminal {
10541050

10551051
pub fn descr(&self) -> &'static str {
10561052
match self {
1057-
NtItem(..) => "item",
10581053
NtBlock(..) => "block",
1059-
NtStmt(..) => "statement",
10601054
NtExpr(..) => "expression",
10611055
NtLiteral(..) => "literal",
10621056
NtMeta(..) => "attribute",
@@ -1078,9 +1072,7 @@ impl PartialEq for Nonterminal {
10781072
impl fmt::Debug for Nonterminal {
10791073
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10801074
match *self {
1081-
NtItem(..) => f.pad("NtItem(..)"),
10821075
NtBlock(..) => f.pad("NtBlock(..)"),
1083-
NtStmt(..) => f.pad("NtStmt(..)"),
10841076
NtExpr(..) => f.pad("NtExpr(..)"),
10851077
NtLiteral(..) => f.pad("NtLiteral(..)"),
10861078
NtMeta(..) => f.pad("NtMeta(..)"),

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
1414
//! ownership of the original.
1515
16-
use crate::ast::{AttrStyle, StmtKind};
16+
use crate::ast::AttrStyle;
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
1818
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
1919
use crate::AttrVec;
@@ -466,13 +466,7 @@ impl TokenStream {
466466

467467
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
468468
match nt {
469-
Nonterminal::NtItem(item) => TokenStream::from_ast(item),
470469
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
471-
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
472-
// FIXME: Properly collect tokens for empty statements.
473-
TokenStream::token_alone(token::Semi, stmt.span)
474-
}
475-
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
476470
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
477471
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
478472
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),

‎compiler/rustc_expand/src/base.rs

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use crate::module::DirOwnership;
55

66
use rustc_ast::attr::MarkedAttrs;
77
use rustc_ast::ptr::P;
8-
use rustc_ast::token::Nonterminal;
8+
use rustc_ast::token::NonterminalKind;
99
use rustc_ast::tokenstream::TokenStream;
1010
use rustc_ast::visit::{AssocCtxt, Visitor};
1111
use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
@@ -15,7 +15,8 @@ use rustc_data_structures::sync::{self, Lrc};
1515
use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed, PResult};
1616
use rustc_feature::Features;
1717
use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools};
18-
use rustc_parse::{parser::Parser, MACRO_ARGUMENTS};
18+
use rustc_parse::parser::{ParseNtResult, Parser};
19+
use rustc_parse::MACRO_ARGUMENTS;
1920
use rustc_session::config::CollapseMacroDebuginfo;
2021
use rustc_session::{parse::ParseSess, Limit, Session};
2122
use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
@@ -1377,13 +1378,13 @@ pub fn parse_macro_name_and_helper_attrs(
13771378
/// If this item looks like a specific enums from `rental`, emit a fatal error.
13781379
/// See #73345 and #83125 for more details.
13791380
/// FIXME(#73933): Remove this eventually.
1380-
fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
1381+
fn pretty_printing_compatibility_hack(item: &Item, psess: &ParseSess) {
13811382
let name = item.ident.name;
13821383
if name == sym::ProceduralMasqueradeDummyType
13831384
&& let ast::ItemKind::Enum(enum_def, _) = &item.kind
13841385
&& let [variant] = &*enum_def.variants
13851386
&& variant.ident.name == sym::Input
1386-
&& let FileName::Real(real) = sess.source_map().span_to_filename(item.ident.span)
1387+
&& let FileName::Real(real) = psess.source_map().span_to_filename(item.ident.span)
13871388
&& let Some(c) = real
13881389
.local_path()
13891390
.unwrap_or(Path::new(""))
@@ -1403,15 +1404,15 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
14031404
if crate_matches {
14041405
// FIXME: make this translatable
14051406
#[allow(rustc::untranslatable_diagnostic)]
1406-
sess.dcx().emit_fatal(errors::ProcMacroBackCompat {
1407+
psess.dcx().emit_fatal(errors::ProcMacroBackCompat {
14071408
crate_name: "rental".to_string(),
14081409
fixed_version: "0.5.6".to_string(),
14091410
});
14101411
}
14111412
}
14121413
}
14131414

1414-
pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &Session) {
1415+
pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, psess: &ParseSess) {
14151416
let item = match ann {
14161417
Annotatable::Item(item) => item,
14171418
Annotatable::Stmt(stmt) => match &stmt.kind {
@@ -1420,17 +1421,34 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &S
14201421
},
14211422
_ => return,
14221423
};
1423-
pretty_printing_compatibility_hack(item, sess)
1424+
pretty_printing_compatibility_hack(item, psess)
14241425
}
14251426

1426-
pub(crate) fn nt_pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &Session) {
1427-
let item = match nt {
1428-
Nonterminal::NtItem(item) => item,
1429-
Nonterminal::NtStmt(stmt) => match &stmt.kind {
1430-
ast::StmtKind::Item(item) => item,
1431-
_ => return,
1432-
},
1427+
pub(crate) fn stream_pretty_printing_compatibility_hack(
1428+
kind: NonterminalKind,
1429+
stream: &TokenStream,
1430+
psess: &ParseSess,
1431+
) {
1432+
let item = match kind {
1433+
NonterminalKind::Item => {
1434+
let mut parser = Parser::new(psess, stream.clone(), None);
1435+
let Ok(ParseNtResult::Item(item)) = parser.parse_nonterminal(kind) else {
1436+
panic!("failed to reparse");
1437+
};
1438+
item
1439+
}
1440+
NonterminalKind::Stmt => {
1441+
// njn: reparsing and then checking for StmtKind::Item sucks, hmm
1442+
let mut parser = Parser::new(psess, stream.clone(), None);
1443+
let Ok(ParseNtResult::Stmt(stmt)) = parser.parse_nonterminal(kind) else {
1444+
panic!("failed to reparse");
1445+
};
1446+
match &stmt.kind {
1447+
ast::StmtKind::Item(item) => item.clone(),
1448+
_ => return,
1449+
}
1450+
}
14331451
_ => return,
14341452
};
1435-
pretty_printing_compatibility_hack(item, sess)
1453+
pretty_printing_compatibility_hack(&item, psess)
14361454
}

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use rustc_ast::mut_visit::{self, MutVisitor};
99
use rustc_ast::token::IdentIsRaw;
1010
use rustc_ast::token::{self, Delimiter, InvisibleOrigin, NonterminalKind, Token, TokenKind};
1111
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
12+
use rustc_ast::StmtKind;
1213
use rustc_data_structures::fx::FxHashMap;
1314
use rustc_errors::{pluralize, Diag, DiagCtxtHandle, PResult};
1415
use rustc_parse::parser::ParseNtResult;
@@ -296,6 +297,18 @@ pub(super) fn transcribe<'a>(
296297
let kind = token::NtLifetime(*ident);
297298
TokenTree::token_alone(kind, sp)
298299
}
300+
MatchedSingle(ParseNtResult::Item(ref item)) => {
301+
mk_delimited(NonterminalKind::Item, TokenStream::from_ast(item))
302+
}
303+
MatchedSingle(ParseNtResult::Stmt(ref stmt)) => {
304+
let stream = if let StmtKind::Empty = stmt.kind {
305+
// FIXME: Properly collect tokens for empty statements.
306+
TokenStream::token_alone(token::Semi, stmt.span)
307+
} else {
308+
TokenStream::from_ast(stmt)
309+
};
310+
mk_delimited(NonterminalKind::Stmt, stream)
311+
}
299312
MatchedSingle(ParseNtResult::PatParam(ref pat, inferred)) => mk_delimited(
300313
NonterminalKind::PatParam { inferred: *inferred },
301314
TokenStream::from_ast(pat),

‎compiler/rustc_expand/src/proc_macro.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ impl MultiItemModifier for DeriveProcMacro {
123123
// We had a lint for a long time, but now we just emit a hard error.
124124
// Eventually we might remove the special case hard error check
125125
// altogether. See #73345.
126-
crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess);
126+
crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess);
127127
let input = item.to_tokens();
128128
let stream = {
129129
let _timer =

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,25 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
114114

115115
while let Some(tree) = cursor.next() {
116116
let (Token { kind, span }, joint) = match tree.clone() {
117-
tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
118-
let delimiter = pm::Delimiter::from_internal(delim);
117+
tokenstream::TokenTree::Delimited(span, _, delim, stream) => {
118+
// We used to have an alternative behaviour for crates that
119+
// needed it: a hack used to pass AST fragments to
120+
// attribute and derive macros as a single nonterminal
121+
// token instead of a token stream. Such token needs to be
122+
// "unwrapped" and not represented as a delimited group. We
123+
// had a lint for a long time, but now we just emit a hard
124+
// error. Eventually we might remove the special case hard
125+
// error check altogether. See #73345.
126+
if let Delimiter::Invisible(InvisibleOrigin::MetaVar(kind)) = delim {
127+
crate::base::stream_pretty_printing_compatibility_hack(
128+
kind,
129+
&stream,
130+
rustc.psess(),
131+
);
132+
}
119133
trees.push(TokenTree::Group(Group {
120-
delimiter,
121-
stream: Some(tts),
134+
delimiter: pm::Delimiter::from_internal(delim),
135+
stream: Some(stream),
122136
span: DelimSpan {
123137
open: span.open,
124138
close: span.close,
@@ -277,15 +291,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
277291

278292
Interpolated(nt) => {
279293
let stream = TokenStream::from_nonterminal_ast(&nt);
280-
// We used to have an alternative behaviour for crates that
281-
// needed it: a hack used to pass AST fragments to
282-
// attribute and derive macros as a single nonterminal
283-
// token instead of a token stream. Such token needs to be
284-
// "unwrapped" and not represented as a delimited group. We
285-
// had a lint for a long time, but now we just emit a hard
286-
// error. Eventually we might remove the special case hard
287-
// error check altogether. See #73345.
288-
crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.ecx.sess);
289294
trees.push(TokenTree::Group(Group {
290295
delimiter: pm::Delimiter::None,
291296
stream: Some(stream),

‎compiler/rustc_parse/src/parser/item.rs

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
use super::diagnostics::{dummy_arg, ConsumeClosingDelim};
22
use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
33
use super::{
4-
AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Trailing, TrailingToken,
4+
AttrWrapper, FollowedByType, ForceCollect, ParseNtResult, Parser, PathStyle, Recovered,
5+
Trailing, TrailingToken,
56
};
67
use crate::errors::{self, MacroExpandsToAdtField};
78
use crate::fluent_generated as fluent;
8-
use crate::maybe_whole;
9+
use crate::maybe_reparse_metavar_seq;
910
use ast::token::IdentIsRaw;
1011
use rustc_ast::ast::*;
1112
use rustc_ast::ptr::P;
@@ -123,10 +124,16 @@ impl<'a> Parser<'a> {
123124
fn_parse_mode: FnParseMode,
124125
force_collect: ForceCollect,
125126
) -> PResult<'a, Option<Item>> {
126-
maybe_whole!(self, NtItem, |item| {
127+
if let Some(mut item) = maybe_reparse_metavar_seq!(
128+
self,
129+
NonterminalKind::Item,
130+
NonterminalKind::Item,
131+
ParseNtResult::Item(item),
132+
item
133+
) {
127134
attrs.prepend_to_nt_inner(&mut item.attrs);
128-
Some(item.into_inner())
129-
});
135+
return Ok(Some(item.into_inner()));
136+
}
130137

131138
let item =
132139
self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1701,6 +1701,8 @@ pub enum ParseNtResult {
17011701
Tt(TokenTree),
17021702
Ident(Ident, IdentIsRaw),
17031703
Lifetime(Ident),
1704+
Item(P<ast::Item>),
1705+
Stmt(P<ast::Stmt>),
17041706
PatParam(P<ast::Pat>, /* inferred */ bool),
17051707
PatWithOr(P<ast::Pat>),
17061708
Ty(P<ast::Ty>),

0 commit comments

Comments
 (0)