Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 666f381

Browse files
nnethercotennethercote
committed
Prepare for invisible delimiters.
Current places where `Interpolated` is used are going to change to instead use invisible delimiters. This prepares for that. - It adds invisible delimiter cases to the `can_begin_*`/`may_be_*` methods that are equivalent to the existing `Interpolated` cases. - It adds panics/asserts in some places where invisible delimiters should never occur. - In `Parser::parse_struct_fields` it excludes an ident + invisible delimiter from special consideration in an error message, because that's quite different to an ident + paren/brace/bracket.
1 parent 2e8c251 commit 666f381

File tree

4 files changed

+99
-14
lines changed

4 files changed

+99
-14
lines changed

‎compiler/rustc_ast/src/token.rs

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -515,10 +515,11 @@ impl Token {
515515

516516
/// Returns `true` if the token can appear at the start of an expression.
517517
pub fn can_begin_expr(&self) -> bool {
518+
use Delimiter::*;
518519
match self.uninterpolate().kind {
519520
Ident(name, is_raw) =>
520521
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
521-
OpenDelim(..) | // tuple, array or block
522+
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
522523
Literal(..) | // literal
523524
Not | // operator not
524525
BinOp(Minus) | // unary minus
@@ -529,13 +530,19 @@ impl Token {
529530
// DotDotDot is no longer supported, but we need some way to display the error
530531
DotDot | DotDotDot | DotDotEq | // range notation
531532
Lt | BinOp(Shl) | // associated path
532-
PathSep | // global path
533+
PathSep | // global path
533534
Lifetime(..) | // labeled loop
534535
Pound => true, // expression attributes
535536
Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
536537
NtExpr(..) |
537538
NtBlock(..) |
538539
NtPath(..)),
540+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
541+
NonterminalKind::Block |
542+
NonterminalKind::Expr(_) |
543+
NonterminalKind::Literal |
544+
NonterminalKind::Path
545+
))) => true,
539546
_ => false,
540547
}
541548
}
@@ -555,11 +562,17 @@ impl Token {
555562
// DotDotDot is no longer supported
556563
| DotDot | DotDotDot | DotDotEq // ranges
557564
| Lt | BinOp(Shl) // associated path
558-
| PathSep => true, // global path
565+
| PathSep => true, // global path
559566
Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
560567
NtPat(..) |
561568
NtBlock(..) |
562569
NtPath(..)),
570+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
571+
NonterminalKind::Block |
572+
NonterminalKind::Pat(_) |
573+
NonterminalKind::Path |
574+
NonterminalKind::Literal
575+
))) => true,
563576
_ => false,
564577
}
565578
}
@@ -580,6 +593,10 @@ impl Token {
580593
Lt | BinOp(Shl) | // associated path
581594
PathSep => true, // global path
582595
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
596+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
597+
NonterminalKind::Ty |
598+
NonterminalKind::Path
599+
))) => true,
583600
// For anonymous structs or unions, which only appear in specific positions
584601
// (type of struct fields or union fields), we don't consider them as regular types
585602
_ => false,
@@ -592,6 +609,9 @@ impl Token {
592609
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
593610
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
594611
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
612+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
613+
NonterminalKind::Expr(_) | NonterminalKind::Block | NonterminalKind::Literal,
614+
))) => true,
595615
_ => false,
596616
}
597617
}
@@ -648,6 +668,10 @@ impl Token {
648668
},
649669
_ => false,
650670
},
671+
// njn: too simple compared to what's above?
672+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
673+
NonterminalKind::Literal | NonterminalKind::Expr(_),
674+
))) => true,
651675
_ => false,
652676
}
653677
}

‎compiler/rustc_parse/src/lexer/tokentrees.rs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,19 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
4242
let mut buf = Vec::new();
4343
loop {
4444
match self.token.kind {
45-
token::OpenDelim(delim) => buf.push(match self.lex_token_tree_open_delim(delim) {
46-
Ok(val) => val,
47-
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
48-
}),
45+
token::OpenDelim(delim) => {
46+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
47+
// code directly from strings, with no macro expansion involved.
48+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
49+
buf.push(match self.lex_token_tree_open_delim(delim) {
50+
Ok(val) => val,
51+
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
52+
})
53+
}
4954
token::CloseDelim(delim) => {
55+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
56+
// code directly from strings, with no macro expansion involved.
57+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
5058
return (
5159
open_spacing,
5260
TokenStream::new(buf),

‎compiler/rustc_parse/src/parser/expr.rs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3572,11 +3572,19 @@ impl<'a> Parser<'a> {
35723572
&& !self.token.is_reserved_ident()
35733573
&& self.look_ahead(1, |t| {
35743574
AssocOp::from_token(t).is_some()
3575-
|| matches!(t.kind, token::OpenDelim(_))
3575+
|| matches!(
3576+
t.kind,
3577+
token::OpenDelim(
3578+
Delimiter::Parenthesis
3579+
| Delimiter::Bracket
3580+
| Delimiter::Brace
3581+
)
3582+
)
35763583
|| t.kind == token::Dot
35773584
})
35783585
{
3579-
// Looks like they tried to write a shorthand, complex expression.
3586+
// Looks like they tried to write a shorthand, complex expression,
3587+
// E.g.: `n + m`, `f(a)`, `a[i]`, `S { x: 3 }`, or `x.y`.
35803588
e.span_suggestion_verbose(
35813589
self.token.span.shrink_to_lo(),
35823590
"try naming a field",

‎compiler/rustc_parse/src/parser/nonterminal.rs

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use rustc_ast::ptr::P;
22
use rustc_ast::token::{
3-
self, Delimiter, Nonterminal::*, NonterminalKind, NtExprKind::*, NtPatKind::*, Token,
3+
self, Delimiter, InvisibleOrigin, Nonterminal, Nonterminal::*, NonterminalKind, NtExprKind::*,
4+
NtPatKind::*, Token,
45
};
56
use rustc_ast::HasTokens;
67
use rustc_ast_pretty::pprust;
@@ -21,7 +22,29 @@ impl<'a> Parser<'a> {
2122
#[inline]
2223
pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
2324
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
24-
fn may_be_ident(nt: &token::Nonterminal) -> bool {
25+
fn may_be_ident(kind: NonterminalKind) -> bool {
26+
use NonterminalKind::*;
27+
match kind {
28+
Stmt
29+
| Pat(_)
30+
| Expr(_)
31+
| Ty
32+
| Literal // `true`, `false`
33+
| Meta
34+
| Path => true,
35+
36+
Item
37+
| Block
38+
| Vis => false,
39+
40+
Ident
41+
| Lifetime
42+
| TT => unreachable!(),
43+
}
44+
}
45+
46+
/// Old variant of `may_be_ident`. Being phased out.
47+
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
2548
match nt {
2649
NtStmt(_)
2750
| NtPat(_)
@@ -59,7 +82,8 @@ impl<'a> Parser<'a> {
5982
| token::Ident(..)
6083
| token::NtIdent(..)
6184
| token::NtLifetime(..)
62-
| token::Interpolated(_) => true,
85+
| token::Interpolated(_)
86+
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
6387
_ => token.can_begin_type(),
6488
},
6589
NonterminalKind::Block => match &token.kind {
@@ -69,11 +93,29 @@ impl<'a> Parser<'a> {
6993
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
7094
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
7195
},
96+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
97+
NonterminalKind::Block
98+
| NonterminalKind::Stmt
99+
| NonterminalKind::Expr(_)
100+
| NonterminalKind::Literal => true,
101+
NonterminalKind::Item
102+
| NonterminalKind::Pat(_)
103+
| NonterminalKind::Ty
104+
| NonterminalKind::Meta
105+
| NonterminalKind::Path
106+
| NonterminalKind::Vis => false,
107+
NonterminalKind::Lifetime | NonterminalKind::Ident | NonterminalKind::TT => {
108+
unreachable!()
109+
}
110+
},
72111
_ => false,
73112
},
74113
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
75114
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
76-
token::Interpolated(nt) => may_be_ident(nt),
115+
token::Interpolated(nt) => nt_may_be_ident(nt),
116+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
117+
may_be_ident(*kind)
118+
}
77119
_ => false,
78120
},
79121
NonterminalKind::Pat(pat_kind) => match &token.kind {
@@ -92,7 +134,10 @@ impl<'a> Parser<'a> {
92134
token::BinOp(token::Shl) => true, // path (double UFCS)
93135
// leading vert `|` or-pattern
94136
token::BinOp(token::Or) => matches!(pat_kind, PatWithOr),
95-
token::Interpolated(nt) => may_be_ident(nt),
137+
token::Interpolated(nt) => nt_may_be_ident(nt),
138+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
139+
may_be_ident(*kind)
140+
}
96141
_ => false,
97142
},
98143
NonterminalKind::Lifetime => match &token.kind {

0 commit comments

Comments
 (0)