Skip to content

Commit eb03a6d

Browse files
nnethercotennethercote
committed
Introduce InvisibleSource on invisible delimiters.
It's not used meaningfully yet, but will be needed to get rid of interpolated tokens.
1 parent e5035df commit eb03a6d

File tree

9 files changed

+51
-25
lines changed

9 files changed

+51
-25
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ impl MetaItemKind {
371371
tokens: &mut impl Iterator<Item = &'a TokenTree>,
372372
) -> Option<MetaItemKind> {
373373
match tokens.next() {
374-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
374+
Some(TokenTree::Delimited(_, Delimiter::Invisible(_), inner_tokens)) => {
375375
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
376376
}
377377
Some(TokenTree::Token(token, _)) => {
@@ -510,7 +510,7 @@ impl NestedMetaItem {
510510
tokens.next();
511511
return Some(NestedMetaItem::Lit(lit));
512512
}
513-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
513+
Some(TokenTree::Delimited(_, Delimiter::Invisible(_), inner_tokens)) => {
514514
tokens.next();
515515
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
516516
}

‎compiler/rustc_ast/src/token.rs

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,33 @@ pub enum Delimiter {
5555
/// "macro variable" `$var`. It is important to preserve operator priorities in cases like
5656
/// `$var * 3` where `$var` is `1 + 2`.
5757
/// Invisible delimiters might not survive roundtrip of a token stream through a string.
58-
Invisible,
58+
Invisible(InvisibleSource),
59+
}
60+
61+
#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
62+
pub enum InvisibleSource {
63+
// Converted from `proc_macro::Delimiter` in
64+
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
65+
ProcMacro,
66+
67+
// Converted from `TokenKind::Interpolated` in
68+
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
69+
FlattenToken,
70+
}
71+
72+
impl Delimiter {
73+
// Should the parser skip these delimiters? Only happens for certain kinds
74+
// of invisible delimiters. Once all interpolated nonterminals are removed,
75+
// the answer should become `false` for all kinds, whereupon this function
76+
// can be removed.
77+
pub fn skip(&self) -> bool {
78+
match self {
79+
Delimiter::Invisible(src) => match src {
80+
InvisibleSource::FlattenToken | InvisibleSource::ProcMacro => true,
81+
},
82+
Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
83+
}
84+
}
5985
}
6086

6187
// Note that the suffix is *not* considered when deciding the `LitKind` in this

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
1616
use crate::ast::{AttrStyle, StmtKind};
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
18-
use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
18+
use crate::token::{self, Delimiter, InvisibleSource, Nonterminal, Token, TokenKind};
1919
use crate::AttrVec;
2020

2121
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -486,7 +486,7 @@ impl TokenStream {
486486
}
487487
token::Interpolated(nt) => TokenTree::Delimited(
488488
DelimSpan::from_single(token.span),
489-
Delimiter::Invisible,
489+
Delimiter::Invisible(InvisibleSource::FlattenToken),
490490
TokenStream::from_nonterminal_ast(nt).flattened(),
491491
),
492492
_ => TokenTree::Token(token.clone(), spacing),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -790,9 +790,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
790790
token::CloseDelim(Delimiter::Bracket) => "]".into(),
791791
token::OpenDelim(Delimiter::Brace) => "{".into(),
792792
token::CloseDelim(Delimiter::Brace) => "}".into(),
793-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
794-
"".into()
795-
}
793+
token::OpenDelim(Delimiter::Invisible(_))
794+
| token::CloseDelim(Delimiter::Invisible(_)) => "".into(),
796795
token::Pound => "#".into(),
797796
token::Dollar => "$".into(),
798797
token::Question => "?".into(),

‎compiler/rustc_expand/src/mbe/macro_rules.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -722,7 +722,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
722722
let mbe::TokenTree::Token(bang) = bang &&
723723
let TokenKind::Not = bang.kind &&
724724
let mbe::TokenTree::Delimited(_, del) = args &&
725-
del.delim != Delimiter::Invisible
725+
!del.delim.skip()
726726
{
727727
true
728728
} else {

‎compiler/rustc_expand/src/mbe/quoted.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,12 @@ fn parse_tree<'a>(
151151
// during parsing.
152152
let mut next = outer_trees.next();
153153
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
154-
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
155-
trees = Box::new(tts.trees());
156-
next = trees.next();
157-
} else {
158-
trees = Box::new(outer_trees);
154+
match next {
155+
Some(tokenstream::TokenTree::Delimited(_, delim, tts)) if delim.skip() => {
156+
trees = Box::new(tts.trees());
157+
next = trees.next();
158+
}
159+
_ => trees = Box::new(outer_trees),
159160
}
160161

161162
match next {

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
3434
token::Delimiter::Parenthesis => Delimiter::Parenthesis,
3535
token::Delimiter::Brace => Delimiter::Brace,
3636
token::Delimiter::Bracket => Delimiter::Bracket,
37-
token::Delimiter::Invisible => Delimiter::None,
37+
token::Delimiter::Invisible(_) => Delimiter::None,
3838
}
3939
}
4040
}
@@ -45,7 +45,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
4545
Delimiter::Parenthesis => token::Delimiter::Parenthesis,
4646
Delimiter::Brace => token::Delimiter::Brace,
4747
Delimiter::Bracket => token::Delimiter::Bracket,
48-
Delimiter::None => token::Delimiter::Invisible,
48+
Delimiter::None => token::Delimiter::Invisible(token::InvisibleSource::ProcMacro),
4949
}
5050
}
5151
}

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ impl TokenCursor {
263263
&TokenTree::Delimited(sp, delim, ref tts) => {
264264
let trees = tts.clone().into_trees();
265265
self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
266-
if delim != Delimiter::Invisible {
266+
if !delim.skip() {
267267
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
268268
}
269269
// No open delimiter to return; continue on to the next iteration.
@@ -272,7 +272,7 @@ impl TokenCursor {
272272
} else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
273273
// We have exhausted this token stream. Move back to its parent token stream.
274274
self.tree_cursor = tree_cursor;
275-
if delim != Delimiter::Invisible {
275+
if !delim.skip() {
276276
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
277277
}
278278
// No close delimiter to return; continue on to the next iteration.
@@ -1046,7 +1046,7 @@ impl<'a> Parser<'a> {
10461046
}
10471047
debug_assert!(!matches!(
10481048
next.0.kind,
1049-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
1049+
token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
10501050
));
10511051
self.inlined_bump_with(next)
10521052
}
@@ -1060,17 +1060,17 @@ impl<'a> Parser<'a> {
10601060
}
10611061

10621062
if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
1063-
&& delim != Delimiter::Invisible
1063+
&& !delim.skip()
10641064
{
10651065
// We are not in the outermost token stream, and the token stream
10661066
// we are in has non-skipped delimiters. Look for skipped
10671067
// delimiters in the lookahead range.
10681068
let tree_cursor = &self.token_cursor.tree_cursor;
1069-
let all_normal = (0..dist).all(|i| {
1069+
let any_skip = (0..dist).any(|i| {
10701070
let token = tree_cursor.look_ahead(i);
1071-
!matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
1071+
matches!(token, Some(TokenTree::Delimited(_, delim, _)) if delim.skip())
10721072
});
1073-
if all_normal {
1073+
if !any_skip {
10741074
// There were no skipped delimiters. Do lookahead by plain indexing.
10751075
return match tree_cursor.look_ahead(dist - 1) {
10761076
Some(tree) => {
@@ -1101,7 +1101,7 @@ impl<'a> Parser<'a> {
11011101
token = cursor.next().0;
11021102
if matches!(
11031103
token.kind,
1104-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
1104+
token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
11051105
) {
11061106
continue;
11071107
}

‎src/tools/rustfmt/src/macros.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -563,7 +563,7 @@ fn delim_token_to_str(
563563
("{ ", " }")
564564
}
565565
}
566-
Delimiter::Invisible => unreachable!(),
566+
Delimiter::Invisible(_) => unreachable!(),
567567
};
568568
if use_multiple_lines {
569569
let indent_str = shape.indent.to_string_with_newline(context.config);

0 commit comments

Comments
 (0)