Merge #4397

4397: Textmate cooperation r=matklad a=georgewfraser

This PR tweaks the fallback TextMate scopes to make them more consistent with the existing grammar and other languages, and edits the builtin TextMate grammar to align with semantic coloring. Before is on the left, after is on the right:

<img width="855" alt="Screen Shot 2020-05-10 at 1 45 51 PM" src="https://user-images.githubusercontent.com/1369240/81512320-a8be7e80-92d4-11ea-8940-2c03f6769015.png">

**Use keyword.other for regular keywords instead of keyword**. This is a really peculiar quirk of TextMate conventions, but virtually *all* TextMate grammars use `keyword.other` (colored blue in VSCode Dark+) for regular keywords and `keyword.control` (colored purple in VSCode Dark+) for control keywords. The TextMate scope `keyword` is colored like control keywords, not regular keywords. It may seem strange that the `keyword` scope is not the right fallback for the `keyword` semantic token, but TextMate has a long and weird history. Note how keywords change from purple back to blue (what they were before semantic coloring was added):

**(1) Use punctuation.section.embedded for format specifiers**. This aligns with how Typescript colors formatting directives:

<img width="238" alt="Screen Shot 2020-05-09 at 10 54 01 AM" src="https://user-images.githubusercontent.com/1369240/81481258-93b5f280-91e3-11ea-99c2-c6d258c5bcad.png">

**(2) Consistently use `entity.name.type.*` scopes for type names**. Avoid using `entity.name.*` which gets colored like a keyword.

**(3) Use Property instead of Member for fields**. Property and Member are very similar, but if you look at the TextMate fallback scopes, it's clear that Member is intended for function-like-things (methods?) and Property is intended for variable-like-things.

**(4) Color `for` as a regular keyword when it's part of `impl Trait for Struct`**. 

**(5) Use `variable.other.constant` for constants instead of `entity.name.constant`**. In the latest VSCode insiders, variable.other.constant has a subtly different color that differentiates constants from ordinary variables. It looks close to the green of types but it's not the same---it's a new color recently added to take advantage of semantic coloring.

I also made some minor changes that make the TextMate scopes better match the semantic scopes. The effect of this for the user is you observe less of a change when semantic coloring "activates". You can see the changes I made relative to the built-in TextMate grammar here:

https://github.com/rust-analyzer/rust-analyzer/pull/4397/files/a91d15c80c337dd1afb0eddd5eb048010d098ac7..97428b6d52d25f810dbd7d7a8d787740c58bfbd2#diff-6966c729b862f79f79bf7258eb3e0885


Co-authored-by: George Fraser <george@fivetran.com>

Author: bors[bot]

crates/ra_ide/src/snapshots/highlighting.html

@@ -33,6 +33,16 @@
     <span class="keyword">pub</span> <span class="field declaration">y</span>: <span class="builtin_type">i32</span>,
 }
 
+<span class="keyword">trait</span> <span class="trait declaration">Bar</span> {
+    <span class="keyword">fn</span> <span class="function declaration">bar</span>(&<span class="keyword">self</span>) -&gt; <span class="builtin_type">i32</span>;
+}
+
+<span class="keyword">impl</span> <span class="trait">Bar</span> <span class="keyword">for</span> <span class="struct">Foo</span> {
+    <span class="keyword">fn</span> <span class="function declaration">bar</span>(&<span class="keyword">self</span>) -&gt; <span class="builtin_type">i32</span> {
+        <span class="keyword">self</span>.<span class="field">x</span>
+    }
+}
+
 <span class="keyword">static</span> <span class="keyword">mut</span> <span class="static declaration mutable">STATIC_MUT</span>: <span class="builtin_type">i32</span> = <span class="numeric_literal">0</span>;
 
 <span class="keyword">fn</span> <span class="function declaration">foo</span>&lt;<span class="lifetime declaration">'a</span>, <span class="type_param declaration">T</span>&gt;() -&gt; <span class="type_param">T</span> {
@@ -63,6 +73,10 @@
         <span class="static mutable">STATIC_MUT</span> = <span class="numeric_literal">1</span>;
     }
 
+    <span class="keyword control">for</span> <span class="variable declaration">e</span> <span class="keyword control">in</span> <span class="variable mutable">vec</span> {
+        <span class="comment">// Do nothing</span>
+    }
+
     <span class="keyword">let</span> <span class="keyword">mut</span> <span class="variable declaration mutable">x</span> = <span class="numeric_literal">42</span>;
     <span class="keyword">let</span> <span class="variable declaration mutable">y</span> = &<span class="keyword">mut</span> <span class="variable mutable">x</span>;
     <span class="keyword">let</span> <span class="variable declaration">z</span> = &<span class="variable mutable">y</span>;

crates/ra_ide/src/syntax_highlighting.rs

@@ -403,12 +403,13 @@ fn highlight_element(
                 T![break]
                 | T![continue]
                 | T![else]
-                | T![for]
                 | T![if]
                 | T![loop]
                 | T![match]
                 | T![return]
-                | T![while] => h | HighlightModifier::ControlFlow,
+                | T![while]
+                | T![in] => h | HighlightModifier::ControlFlow,
+                T![for] if !is_child_of_impl(element) => h | HighlightModifier::ControlFlow,
                 T![unsafe] => h | HighlightModifier::Unsafe,
                 _ => h,
             }
@@ -432,6 +433,13 @@ fn highlight_element(
     }
 }
 
+fn is_child_of_impl(element: SyntaxElement) -> bool {
+    match element.parent() {
+        Some(e) => e.kind() == IMPL_DEF,
+        _ => false,
+    }
+}
+
 fn highlight_name(db: &RootDatabase, def: Definition) -> Highlight {
     match def {
         Definition::Macro(_) => HighlightTag::Macro,

crates/ra_ide/src/syntax_highlighting/tests.rs

@@ -17,6 +17,16 @@ struct Foo {
     pub y: i32,
 }
 
+trait Bar {
+    fn bar(&self) -> i32;
+}
+
+impl Bar for Foo {
+    fn bar(&self) -> i32 {
+        self.x
+    }
+}
+
 static mut STATIC_MUT: i32 = 0;
 
 fn foo<'a, T>() -> T {
@@ -47,6 +57,10 @@ fn main() {
         STATIC_MUT = 1;
     }
 
+    for e in vec {
+        // Do nothing
+    }
+
     let mut x = 42;
     let y = &mut x;
     let z = &y;

crates/rust-analyzer/src/to_proto.rs

@@ -264,7 +264,7 @@ fn semantic_token_type_and_modifiers(
         HighlightTag::Trait => lsp_types::SemanticTokenType::INTERFACE,
         HighlightTag::BuiltinType => semantic_tokens::BUILTIN_TYPE,
         HighlightTag::SelfType => lsp_types::SemanticTokenType::TYPE,
-        HighlightTag::Field => lsp_types::SemanticTokenType::MEMBER,
+        HighlightTag::Field => lsp_types::SemanticTokenType::PROPERTY,
         HighlightTag::Function => lsp_types::SemanticTokenType::FUNCTION,
         HighlightTag::Module => lsp_types::SemanticTokenType::NAMESPACE,
         HighlightTag::Constant => {

editors/code/.vscodeignore

@@ -3,5 +3,6 @@
 !package.json
 !package-lock.json
 !ra_syntax_tree.tmGrammar.json
+!rust.tmGrammar.json
 !icon.png
 !README.md

editors/code/package.json

@@ -478,6 +478,11 @@
             }
         ],
         "grammars": [
+            {
+                "language": "rust",
+                "scopeName": "source.rust",
+                "path": "rust.tmGrammar.json"
+            },
             {
                 "language": "ra_syntax_tree",
                 "scopeName": "source.ra_syntax_tree",
@@ -596,28 +601,28 @@
                         "support.type.primitive"
                     ],
                     "lifetime": [
-                        "entity.name.lifetime.rust"
+                        "storage.modifier.lifetime.rust"
                     ],
                     "typeAlias": [
-                        "entity.name.typeAlias"
+                        "entity.name.type.typeAlias"
                     ],
                     "union": [
-                        "entity.name.union"
+                        "entity.name.type.union"
                     ],
                     "struct": [
                         "entity.name.type.struct"
                     ],
-                    "keyword.unsafe": [
-                        "keyword.other.unsafe"
-                    ],
                     "keyword": [
-                        "keyword"
+                        "keyword.other"
                     ],
                     "keyword.controlFlow": [
                         "keyword.control"
                     ],
                     "variable.constant": [
-                        "entity.name.constant"
+                        "variable.other.constant"
+                    ],
+                    "formatSpecifier": [
+                        "punctuation.section.embedded"
                     ]
                 }
             }