Fix invalid color syntax generating color tokens

Author: duckdoom4

examples/tokenizer_tests.rpy

@@ -21,4 +21,13 @@ init python hide early in Namespace:
             self.items.append(item);
             return
 
+label start:
+    "Light Distraction\n{color=ff0000}{size=18}[distraction_percentages[0]]%% Chance{/size}{/color}"
+    
+    menu:    
+        "It's a video game.":
+            jump game
+
+        "It's an interactive book.":
+            jump book
 

src/tokenizer/atl-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/06/2023 21:08:08 (UTC+0)
+// Last generated: 14/06/2023 16:51:20 (UTC+0)
 
 import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";

src/tokenizer/python-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/06/2023 21:08:08 (UTC+0)
+// Last generated: 14/06/2023 16:51:20 (UTC+0)
 
 import { MetaTokenType, KeywordTokenType, EntityTokenType, CharacterTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";

src/tokenizer/renpy-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/06/2023 21:08:08 (UTC+0)
+// Last generated: 14/06/2023 16:51:20 (UTC+0)
 
 import { MetaTokenType, CharacterTokenType, LiteralTokenType, EntityTokenType, KeywordTokenType, EscapedCharacterTokenType, OperatorTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -431,7 +431,6 @@ export const escapedUnicodeChar: TokenPattern = {
 };
 
 export const hexLiteral: TokenPattern = {
-    // Note: This pattern has no end check. Only use as include pattern!
     patterns: [
         {
             debugName: "hexLiteral.patterns![0]",
@@ -449,10 +448,10 @@ export const hexLiteral: TokenPattern = {
         {
             debugName: "hexLiteral.patterns![2]",
 
-            token: LiteralTokenType.Color, /*constant.color.renpy*/
-            match: /(?:#[a-f0-9]*)?(.+)/dgi,
+            match: /(#[a-f0-9]*)?(.+?)\b/dgi,
             captures: {
-                1: { token: MetaTokenType.Invalid, /*invalid.illegal.character-not-allowed-here.renpy*/ },
+                1: { token: LiteralTokenType.Color, /*constant.color.renpy*/ },
+                2: { token: MetaTokenType.Invalid, /*invalid.illegal.character-not-allowed-here.renpy*/ },
             },
         },
     ]
@@ -573,10 +572,7 @@ export const stringTagsBroken: TokenPattern = {
                 1: { token: CharacterTokenType.OpenBracket, /*punctuation.definition.tag.begin.renpy*/ },
                 2: { token: EntityTokenType.TagName, /*entity.name.tag.${2:/downcase}.renpy*/ },
                 3: { token: OperatorTokenType.Assignment, /*punctuation.separator.key-value.renpy keyword.operator.assignment.renpy*/ },
-                4: {
-                    token: LiteralTokenType.Color, /*constant.color.renpy*/
-                    patterns: [hexLiteral]
-                },
+                4: { patterns: [hexLiteral] },
                 5: { token: CharacterTokenType.CloseBracket, /*punctuation.definition.tag.end.renpy*/ },
             },
             end: /({\/)\s*(\2)\s*(})/dg,
@@ -758,10 +754,7 @@ export const stringTags: TokenPattern = {
                 1: { token: CharacterTokenType.OpenBracket, /*punctuation.definition.tag.begin.renpy*/ },
                 2: { token: EntityTokenType.TagName, /*entity.name.tag.${2:/downcase}.renpy*/ },
                 3: { token: OperatorTokenType.Assignment, /*punctuation.separator.key-value.renpy keyword.operator.assignment.renpy*/ },
-                4: {
-                    token: LiteralTokenType.Color, /*constant.color.renpy*/
-                    patterns: [hexLiteral]
-                },
+                4: { patterns: [hexLiteral] },
                 5: { token: CharacterTokenType.CloseBracket, /*punctuation.definition.tag.end.renpy*/ },
             },
         },

src/tokenizer/screen-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/06/2023 21:08:08 (UTC+0)
+// Last generated: 14/06/2023 16:51:20 (UTC+0)
 
 import { EntityTokenType, CharacterTokenType, KeywordTokenType, MetaTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";

src/tokenizer/token-patterns.g.ts

@@ -3,12 +3,12 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/06/2023 21:08:08 (UTC+0)
+// Last generated: 14/06/2023 16:51:20 (UTC+0)
 
-import * as RenpyPatterns from "./renpy-token-patterns.g";
+import * as PythonPatterns from "./python-token-patterns.g";
 import * as ScreenPatterns from "./screen-token-patterns.g";
 import * as AtlPatterns from "./atl-token-patterns.g";
-import * as PythonPatterns from "./python-token-patterns.g";
+import * as RenpyPatterns from "./renpy-token-patterns.g";
 
 // Push all RenpyPatterns external includes
 RenpyPatterns.literal.patterns!.push(PythonPatterns.literal);
@@ -96,4 +96,4 @@ ScreenPatterns.screenLanguage.patterns!.splice(0, 0, RenpyPatterns.conditionals)
 PythonPatterns.stringUnicodeGuts.patterns!.splice(0, 0, RenpyPatterns.stringsInterior);
 
 
-export { RenpyPatterns, ScreenPatterns, AtlPatterns, PythonPatterns };
+export { PythonPatterns, ScreenPatterns, AtlPatterns, RenpyPatterns };

syntaxes/renpy.tmLanguage.json

@@ -329,7 +329,6 @@
       }
     },
     "hex_literal": {
-      "comment": "Note: This pattern has no end check. Only use as include pattern!",
       "patterns": [
         {
           "comment": "rgb, rgba, rrggbb, rrggbbaa",
@@ -341,10 +340,10 @@
           "name": "invalid.illegal.unexpected-number-of-hex-values.renpy"
         },
         {
-          "match": "(?i)(?:#[a-f0-9]*)?(.+)",
-          "name": "constant.color.renpy",
+          "match": "(?i)(#[a-f0-9]*)?(.+?)\\b",
           "captures": {
-            "1": { "name": "invalid.illegal.character-not-allowed-here.renpy" }
+            "1": { "name": "constant.color.renpy" },
+            "2": { "name": "invalid.illegal.character-not-allowed-here.renpy" }
           }
         }
       ]
@@ -450,7 +449,6 @@
             "2": { "name": "entity.name.tag.${2:/downcase}.renpy" },
             "3": { "name": "punctuation.separator.key-value.renpy keyword.operator.assignment.renpy" },
             "4": {
-              "name": "constant.color.renpy",
               "patterns": [{ "include": "#hex_literal" }]
             },
             "5": { "name": "punctuation.definition.tag.end.renpy" }
@@ -611,7 +609,6 @@
             "2": { "name": "entity.name.tag.${2:/downcase}.renpy" },
             "3": { "name": "punctuation.separator.key-value.renpy keyword.operator.assignment.renpy" },
             "4": {
-              "name": "constant.color.renpy",
               "patterns": [{ "include": "#hex_literal" }]
             },
             "5": { "name": "punctuation.definition.tag.end.renpy" }