Proper span representation with syntax context

Author: Veykril

crates/base-db/src/span.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 
 use salsa::InternId;
+use tt::SyntaxContext;
 use vfs::FileId;
 
 pub type ErasedFileAstId = la_arena::Idx<syntax::SyntaxNodePtr>;
@@ -9,10 +10,17 @@ pub type ErasedFileAstId = la_arena::Idx<syntax::SyntaxNodePtr>;
 pub const ROOT_ERASED_FILE_AST_ID: ErasedFileAstId =
     la_arena::Idx::from_raw(la_arena::RawIdx::from_u32(0));
 
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
-pub struct SyntaxContext;
+pub type SpanData = tt::SpanData<SpanAnchor, SyntaxContextId>;
 
-pub type SpanData = tt::SpanData<SpanAnchor>;
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SyntaxContextId(InternId);
+crate::impl_intern_key!(SyntaxContextId);
+
+impl SyntaxContext for SyntaxContextId {
+    // FIXME: This is very much UB, salsa exposes no way to create an InternId in a const context
+    // currently (which kind of makes sense but we need it here!)
+    const DUMMY: Self = SyntaxContextId(unsafe { core::mem::transmute(1) });
+}
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 pub struct SpanAnchor {
@@ -26,7 +34,7 @@ impl fmt::Debug for SpanAnchor {
     }
 }
 
-impl tt::Span for SpanAnchor {
+impl tt::SpanAnchor for SpanAnchor {
     const DUMMY: Self = SpanAnchor { file_id: HirFileId(0), ast_id: ROOT_ERASED_FILE_AST_ID };
 }
 

crates/cfg/src/tests.rs

@@ -2,28 +2,43 @@ use arbitrary::{Arbitrary, Unstructured};
 use expect_test::{expect, Expect};
 use mbe::syntax_node_to_token_tree;
 use syntax::{ast, AstNode};
-use tt::Span;
+use tt::{SpanAnchor, SyntaxContext};
 
 use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr};
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 struct DummyFile;
-impl Span for DummyFile {
+impl SpanAnchor for DummyFile {
     const DUMMY: Self = DummyFile;
 }
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+struct DummyCtx;
+impl SyntaxContext for DummyCtx {
+    const DUMMY: Self = DummyCtx;
+}
 
 fn assert_parse_result(input: &str, expected: CfgExpr) {
     let source_file = ast::SourceFile::parse(input).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default());
+    let tt = syntax_node_to_token_tree::<_, DummyCtx>(
+        tt.syntax(),
+        DummyFile,
+        0.into(),
+        &Default::default(),
+    );
     let cfg = CfgExpr::parse(&tt);
     assert_eq!(cfg, expected);
 }
 
 fn check_dnf(input: &str, expect: Expect) {
     let source_file = ast::SourceFile::parse(input).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default());
+    let tt = syntax_node_to_token_tree::<_, DummyCtx>(
+        tt.syntax(),
+        DummyFile,
+        0.into(),
+        &Default::default(),
+    );
     let cfg = CfgExpr::parse(&tt);
     let actual = format!("#![cfg({})]", DnfExpr::new(cfg));
     expect.assert_eq(&actual);
@@ -32,7 +47,12 @@ fn check_dnf(input: &str, expect: Expect) {
 fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) {
     let source_file = ast::SourceFile::parse(input).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default());
+    let tt = syntax_node_to_token_tree::<_, DummyCtx>(
+        tt.syntax(),
+        DummyFile,
+        0.into(),
+        &Default::default(),
+    );
     let cfg = CfgExpr::parse(&tt);
     let dnf = DnfExpr::new(cfg);
     let why_inactive = dnf.why_inactive(opts).unwrap().to_string();
@@ -43,7 +63,12 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) {
 fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) {
     let source_file = ast::SourceFile::parse(input).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt = syntax_node_to_token_tree(tt.syntax(), DummyFile, 0.into(), &Default::default());
+    let tt = syntax_node_to_token_tree::<_, DummyCtx>(
+        tt.syntax(),
+        DummyFile,
+        0.into(),
+        &Default::default(),
+    );
     let cfg = CfgExpr::parse(&tt);
     let dnf = DnfExpr::new(cfg);
     let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::<Vec<_>>();

crates/hir-def/src/attr/tests.rs

@@ -4,15 +4,25 @@
 use base_db::span::SpanAnchor;
 use mbe::syntax_node_to_token_tree;
 use syntax::{ast, AstNode};
-use tt::Span;
+use tt::{SpanAnchor as _, SyntaxContext};
 
 use crate::attr::{DocAtom, DocExpr};
 
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+struct DummyCtx;
+impl SyntaxContext for DummyCtx {
+    const DUMMY: Self = DummyCtx;
+}
+
 fn assert_parse_result(input: &str, expected: DocExpr) {
     let source_file = ast::SourceFile::parse(input).ok().unwrap();
     let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-    let tt =
-        syntax_node_to_token_tree(tt.syntax(), SpanAnchor::DUMMY, 0.into(), &Default::default());
+    let tt = syntax_node_to_token_tree::<_, DummyCtx>(
+        tt.syntax(),
+        SpanAnchor::DUMMY,
+        0.into(),
+        &Default::default(),
+    );
     let cfg = DocExpr::parse(&tt);
     assert_eq!(cfg, expected);
 }

crates/hir-expand/src/attrs.rs

@@ -1,7 +1,7 @@
 //! A higher level attributes based on TokenTree, with also some shortcuts.
 use std::{fmt, ops};
 
-use ::tt::Span;
+use ::tt::SpanAnchor as _;
 use base_db::{span::SpanAnchor, CrateId};
 use cfg::CfgExpr;
 use either::Either;

crates/hir-expand/src/builtin_fn_macro.rs

@@ -1,5 +1,6 @@
 //! Builtin macro
 
+use ::tt::Span;
 use base_db::{
     span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID},
     AnchoredPath, Edition, FileId,
@@ -15,7 +16,7 @@ use syntax::{
 use crate::{
     db::ExpandDatabase,
     name, quote,
-    tt::{self, Span},
+    tt::{self},
     EagerCallInfo, ExpandError, ExpandResult, HirFileIdExt, MacroCallId, MacroCallLoc,
 };
 

crates/hir-expand/src/db.rs

@@ -1,8 +1,9 @@
 //! Defines database & queries for macro expansion.
 
+use ::tt::SyntaxContext;
 use base_db::{
     salsa,
-    span::{SpanAnchor, ROOT_ERASED_FILE_AST_ID},
+    span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID},
     CrateId, Edition, SourceDatabase,
 };
 use either::Either;
@@ -15,11 +16,13 @@ use syntax::{
 use triomphe::Arc;
 
 use crate::{
-    ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion,
-    builtin_fn_macro::EagerExpander, hygiene::HygieneFrame, tt, AstId, BuiltinAttrExpander,
-    BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult,
-    ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId,
-    MacroDefKind, MacroFile, ProcMacroExpander, SpanMap, SyntaxContext, SyntaxContextId,
+    ast_id_map::AstIdMap,
+    builtin_attr_macro::pseudo_derive_attr_expansion,
+    builtin_fn_macro::EagerExpander,
+    hygiene::{self, HygieneFrame, SyntaxContextData},
+    tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo,
+    ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind,
+    MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, SpanMap,
 };
 
 /// Total limit on the number of tokens produced by any macro invocation.
@@ -89,7 +92,15 @@ pub trait ExpandDatabase: SourceDatabase {
     #[salsa::interned]
     fn intern_macro_call(&self, macro_call: MacroCallLoc) -> MacroCallId;
     #[salsa::interned]
-    fn intern_syntax_context(&self, ctx: SyntaxContext) -> SyntaxContextId;
+    fn intern_syntax_context(&self, ctx: SyntaxContextData) -> SyntaxContextId;
+    #[salsa::transparent]
+    #[salsa::invoke(hygiene::apply_mark)]
+    fn apply_mark(
+        &self,
+        ctxt: SyntaxContextData,
+        file_id: HirFileId,
+        transparency: hygiene::Transparency,
+    ) -> SyntaxContextId;
 
     /// Lowers syntactic macro call to a token tree representation. That's a firewall
     /// query, only typing in the macro call itself changes the returned
@@ -225,6 +236,7 @@ pub fn expand_speculative(
         .ranges_with_span(tt::SpanData {
             range: token_to_map.text_range(),
             anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
+            ctx: SyntaxContextId::DUMMY,
         })
         .filter_map(|range| syntax_node.covering_element(range).into_token())
         .min_by_key(|t| {