Auto merge of #57367 - petrochenkov:unrestab, r=Centril

Stabilize `unrestricted_attribute_tokens`

In accordance with a plan described in https://internals.rust-lang.org/t/unrestricted-attribute-tokens-feature-status/8561/3.

Delimited non-macro non-builtin attributes now support the same syntax as macro attributes:
```
PATH
PATH `(` TOKEN_STREAM `)`
PATH `[` TOKEN_STREAM `]`
PATH `{` TOKEN_STREAM `}`
```
Such attributes mostly serve as inert proc macro helpers or tool attributes.
To some extent these attributes are de-facto stable due to a hole in feature gate checking (feature gating is done too late - after macro expansion.)
So if macro *removes* such helper attributes during expansion (and it must remove them, unless it's a derive macro), then the code will work on stable.

Key-value non-macro non-builtin attributes are now restricted to bare minimum required to support what we support on stable - unsuffixed literals (#34981).
```
PATH `=` LITERAL
```
(Key-value macro attributes are not supported at all right now.)
Crater run in #57321 found no regressions for this change.
There are multiple possible ways to extend key-value attributes (#57321 (comment)), but I'd expect an RFC for that and it's not a pressing enough issue to block stabilization of delimited attributes.

Built-in attributes are still restricted to the "classic" meta-item syntax, nothing changes here.
#57321 goes further and adds some additional restrictions (more consistent input checking) to built-in attributes.

Closes #55208

Author: bors

src/libcore/lib.rs

@@ -123,7 +123,6 @@
 #![feature(abi_unadjusted)]
 #![feature(adx_target_feature)]
 #![feature(maybe_uninit, maybe_uninit_slice, maybe_uninit_array)]
-#![feature(unrestricted_attribute_tokens)]
 #![feature(external_doc)]
 
 #[prelude_import]

src/libsyntax/feature_gate.rs

@@ -21,8 +21,9 @@ use crate::early_buffered_lints::BufferedEarlyLintId;
 use crate::source_map::Spanned;
 use crate::edition::{ALL_EDITIONS, Edition};
 use crate::visit::{self, FnKind, Visitor};
-use crate::parse::ParseSess;
+use crate::parse::{token, ParseSess};
 use crate::symbol::Symbol;
+use crate::tokenstream::TokenTree;
 
 use errors::{DiagnosticBuilder, Handler};
 use rustc_data_structures::fx::FxHashMap;
@@ -431,9 +432,6 @@ declare_features! (
     // Added for testing E0705; perma-unstable.
     (active, test_2018_feature, "1.31.0", Some(0), Some(Edition::Edition2018)),
 
-    // support for arbitrary delimited token streams in non-macro attributes
-    (active, unrestricted_attribute_tokens, "1.30.0", Some(55208), None),
-
     // Allows unsized rvalues at arguments and parameters.
     (active, unsized_locals, "1.30.0", Some(48055), None),
 
@@ -700,6 +698,8 @@ declare_features! (
     (accepted, cfg_target_vendor, "1.33.0", Some(29718), None),
     // `extern crate self as foo;` puts local crate root into extern prelude under name `foo`.
     (accepted, extern_crate_self, "1.34.0", Some(56409), None),
+    // support for arbitrary delimited token streams in non-macro attributes
+    (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208), None),
 );
 
 // If you change this, please modify `src/doc/unstable-book` as well. You must
@@ -1660,13 +1660,9 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
         match BUILTIN_ATTRIBUTES.iter().find(|(name, ..)| attr.path == name) {
             Some(&(name, _, template, _)) => self.check_builtin_attribute(attr, name, template),
-            None => if !self.context.features.unrestricted_attribute_tokens {
-                // Unfortunately, `parse_meta` cannot be called speculatively
-                // because it can report errors by itself, so we have to call it
-                // only if the feature is disabled.
-                if let Err(mut err) = attr.parse_meta(self.context.parse_sess) {
-                    err.help("try enabling `#![feature(unrestricted_attribute_tokens)]`").emit()
-                }
+            None => if let Some(TokenTree::Token(_, token::Eq)) = attr.tokens.trees().next() {
+                // All key-value attributes are restricted to meta-item syntax.
+                attr.parse_meta(self.context.parse_sess).map_err(|mut err| err.emit()).ok();
             }
         }
     }

src/libsyntax/parse/attr.rs

@@ -158,11 +158,21 @@ impl<'a> Parser<'a> {
                    self.parse_token_tree().into()
             } else if self.eat(&token::Eq) {
                 let eq = TokenTree::Token(self.prev_span, token::Eq);
-                let tree = match self.token {
-                    token::CloseDelim(_) | token::Eof => self.unexpected()?,
-                    _ => self.parse_token_tree(),
+                let mut is_interpolated_expr = false;
+                if let token::Interpolated(nt) = &self.token {
+                    if let token::NtExpr(..) = **nt {
+                        is_interpolated_expr = true;
+                    }
+                }
+                let tokens = if is_interpolated_expr {
+                    // We need to accept arbitrary interpolated expressions to continue
+                    // supporting things like `doc = $expr` that work on stable.
+                    // Non-literal interpolated expressions are rejected after expansion.
+                    self.parse_token_tree().into()
+                } else {
+                    self.parse_unsuffixed_lit()?.tokens()
                 };
-                TokenStream::new(vec![eq.into(), tree.into()])
+                TokenStream::from_streams(vec![eq.into(), tokens])
             } else {
                 TokenStream::empty()
             };

src/libsyntax/tokenstream.rs

@@ -254,7 +254,7 @@ impl TokenStream {
         }
     }
 
-    fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream {
+    pub(crate) fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
             1 => streams.pop().unwrap(),

src/test/run-pass/proc-macro/derive-b.rs

@@ -1,7 +1,5 @@
 // aux-build:derive-b.rs
 
-#![feature(unrestricted_attribute_tokens)]
-
 extern crate derive_b;
 
 #[derive(Debug, PartialEq, derive_b::B, Eq, Copy, Clone)]

src/test/ui/attr-eq-token-tree.rs

@@ -1,6 +1,4 @@
-// compile-pass
+#![feature(custom_attribute)]
 
-#![feature(custom_attribute, unrestricted_attribute_tokens)]
-
-#[my_attr = !] // OK under feature gate
+#[my_attr = !] //~ ERROR unexpected token: `!`
 fn main() {}

src/test/ui/attr-eq-token-tree.stderr

@@ -0,0 +1,8 @@
+error: unexpected token: `!`
+  --> $DIR/attr-eq-token-tree.rs:3:11
+   |
+LL | #[my_attr = !] //~ ERROR unexpected token: `!`
+   |           ^
+
+error: aborting due to previous error
+

src/test/ui/feature-gates/feature-gate-unrestricted-attribute-tokens.rs

@@ -1,4 +1,2 @@
-#![feature(unrestricted_attribute_tokens)]
-
-#[doc = $not_there] //~ ERROR expected `]`, found `not_there`
+#[doc = $not_there] //~ ERROR unexpected token: `$`
 fn main() { }