Merge pull request #444 from renpy/parser-pattern-fixes

Parser pattern fixes

Author: duckdoom4

src/tokenizer/renpy-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 15/08/2024 21:19:25 (UTC+0)
+// Last generated: 17/08/2024 11:27:04 (UTC+0)
 
 import { MetaTokenType, CharacterTokenType, LiteralTokenType, EntityTokenType, KeywordTokenType, EscapedCharacterTokenType, OperatorTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -1033,7 +1033,7 @@ export const sayStatements: TokenPattern = {
 
             token: MetaTokenType.SayStatement, /*meta.say.statement.renpy*/
             contentToken: LiteralTokenType.String, /*renpy.meta.say.$1 string.quoted.renpy*/
-            begin: /(?<=^[ \t]+)(?:([a-zA-Z_]\w*)\b|"([a-zA-Z_]\w*)\b")((?:[ \t]+(?:@|\w+))*)?[ \t]*("""|"|'''|'|```|`)/dgm,
+            begin: /(?<=^[ \t]+)(?:((?:character\.)?[a-zA-Z_]\w*)\b|"([a-zA-Z_]\w*)\b")((?:[ \t]+(?:@|\w+))*)?(?=[ \t]*r?(?:"|'|`))/dgm,
             beginCaptures: {
                 1: {
                     token: EntityTokenType.Identifier, /*renpy.meta.character.$1 variable.other.renpy*/
@@ -1073,18 +1073,11 @@ export const sayStatements: TokenPattern = {
                 },
                 4: { token: MetaTokenType.StringBegin, /*string.quoted.renpy punctuation.definition.string.begin.renpy*/ },
             },
-            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
-            end: /(?<![^\\]\\)(((?<=\4)\4)|\4)([ \t]*\(.*?\)(?![^\(]*?\)))?/dg,
-            endCaptures: {
-                1: { token: MetaTokenType.StringEnd, /*string.quoted.renpy punctuation.definition.string.end.renpy*/ },
-                2: { token: MetaTokenType.EmptyString, /*meta.empty-string.renpy*/ },
-                3: {
-                    token: MetaTokenType.Arguments, /*meta.say.arguments.renpy*/
-                    patterns: [
-                    ]
-                },
-            },
-            patterns: [stringsInterior]
+            end: /$/gm,
+            patterns: [
+                strings,
+                fallbackPatterns,
+            ]
         },
         {
             debugName: "sayStatements.patterns![1]",
@@ -1237,23 +1230,35 @@ export const image: TokenPattern = {
             debugName: "image.patterns![0]",
 
             token: MetaTokenType.ImageStatement, /*meta.image.statement.renpy*/
-            begin: /(?<=^[ \t]*)(image)\b[ \t]*/dgm,
+            begin: /(?<=^[ \t]*)(image)\b/dgm,
             beginCaptures: {
                 1: { token: KeywordTokenType.Image, /*keyword.image.renpy*/ },
             },
-            end: /(?=\b(at|with)\b|#|=|:)|$/gm,
+            end: /(?=:)|$/gm,
             patterns: [
                 strings,
                 {
                     debugName: "image.patterns![0].patterns![1]",
 
                     token: EntityTokenType.ImageName, /*entity.name.type.image.renpy*/
-                    match: /\b(?:[a-zA-Z_0-9]+)\b[ \t]*/g,
+                    match: /\b(?:[a-zA-Z_0-9]+)\b/g,
                 },
+                atStatement,
+                withStatement,
+                {
+                    debugName: "image.patterns![0].patterns![4]",
+
+                    contentToken: MetaTokenType.PythonExpression, /*meta.python.expression.renpy*/
+                    begin: /=/dg,
+                    beginCaptures: {
+                        0: { token: OperatorTokenType.Assignment, /*keyword.operator.assignment.renpy*/ },
+                    },
+                    end: /$/gm,
+                    patterns: [whitespace]
+                },
+                fallbackPatterns,
             ]
         },
-        atStatement,
-        withStatement,
     ]
 };
 
@@ -1681,7 +1686,7 @@ export const label: TokenPattern = {
     beginCaptures: {
         1: { token: KeywordTokenType.Label, /*storage.type.function.label.renpy*/ },
     },
-    end: /(?=#|:)|$/gm,
+    end: /(?=:)|$/gm,
     patterns: [
         {
             debugName: "label.patterns![0]",
@@ -1690,6 +1695,7 @@ export const label: TokenPattern = {
             match: /\b(?<!\.)(hide)\b/g,
         },
         labelDefName,
+        fallbackPatterns,
     ]
 };
 
@@ -2054,6 +2060,7 @@ simpleExpression.patterns!.splice(7, 0, comments);
 stringsInterior.patterns!.splice(2, 0, stringTags);
 stringTags.patterns![10].captures![4].patterns![0].captures![3].patterns!.splice(0, 0, labelName);
 pythonStatements.patterns!.push(define, defaultStatement, oneLinePython, pythonBlockTester);
+sayStatements.patterns![0].patterns!.splice(2, 0, withStatement);
 sayStatements.patterns![1].patterns!.splice(2, 0, withStatement);
 renpyBlockTester.patterns!.push(basePatterns);
 layeredimageBlockTester.patterns!.push(layeredimageGroup, layeredimageAttribute, basePatterns);

src/tokenizer/token-patterns.g.ts

@@ -3,11 +3,11 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 15/08/2024 20:21:42 (UTC+0)
+// Last generated: 17/08/2024 11:27:04 (UTC+0)
 
-import * as AtlPatterns from "./atl-token-patterns.g";
 import * as RenpyPatterns from "./renpy-token-patterns.g";
 import * as PythonPatterns from "./python-token-patterns.g";
+import * as AtlPatterns from "./atl-token-patterns.g";
 import * as ScreenPatterns from "./screen-token-patterns.g";
 import * as StylePatterns from "./style-token-patterns.g";
 
@@ -28,14 +28,15 @@ RenpyPatterns.define.patterns![3].patterns!.splice(0, 0, PythonPatterns.expressi
 RenpyPatterns.defaultStatement.patterns!.splice(1, 0, PythonPatterns.expression);
 RenpyPatterns.defaultStatement.patterns![2].patterns!.splice(0, 0, PythonPatterns.expression);
 RenpyPatterns.oneLinePython.patterns!.splice(1, 0, PythonPatterns.expression);
-RenpyPatterns.sayStatements.patterns![0].endCaptures![3].patterns!.push(PythonPatterns.oddFunctionCall);
+RenpyPatterns.sayStatements.patterns![0].patterns!.splice(1, 0, PythonPatterns.oddFunctionCall);
 RenpyPatterns.sayStatements.patterns![1].patterns!.splice(1, 0, PythonPatterns.oddFunctionCall);
 RenpyPatterns.conditionals.patterns!.splice(0, 0, PythonPatterns.expression);
+RenpyPatterns.image.patterns![0].patterns![4].patterns!.splice(0, 0, PythonPatterns.expression);
 RenpyPatterns.labelName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables);
 RenpyPatterns.labelCall.patterns!.splice(0, 0, PythonPatterns.specialVariables);
 RenpyPatterns.labelCall.patterns!.push(PythonPatterns.functionArguments);
 RenpyPatterns.labelDefName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables);
-RenpyPatterns.label.patterns!.push(PythonPatterns.parameters);
+RenpyPatterns.label.patterns!.splice(2, 0, PythonPatterns.parameters);
 RenpyPatterns.returnStatements.patterns!.push(PythonPatterns.expression);
 RenpyPatterns.callJumpExpression.patterns!.push(PythonPatterns.expression);
 RenpyPatterns.callPass.patterns!.splice(0, 0, PythonPatterns.functionArguments);
@@ -145,4 +146,4 @@ PythonPatterns.doubleOneRegexpComments.patterns!.push(RenpyPatterns.codetags);
 PythonPatterns.doubleThreeRegexpComments.patterns!.push(RenpyPatterns.codetags);
 
 
-export { AtlPatterns, RenpyPatterns, PythonPatterns, ScreenPatterns, StylePatterns };
+export { RenpyPatterns, PythonPatterns, AtlPatterns, ScreenPatterns, StylePatterns };

syntaxes/renpy.tmLanguage.json

@@ -851,7 +851,7 @@
         {
           "name": "meta.say.statement.renpy",
           "contentName": "renpy.meta.say.$1 string.quoted.renpy",
-          "begin": "(?<=^[ \\t]+)(?:([a-zA-Z_]\\w*)\\b|\"([a-zA-Z_]\\w*)\\b\")((?:[ \\t]+(?:@|\\w+))*)?[ \\t]*(\"\"\"|\"|'''|'|```|`)",
+          "begin": "(?<=^[ \\t]+)(?:((?:character\\.)?[a-zA-Z_]\\w*)\\b|\"([a-zA-Z_]\\w*)\\b\")((?:[ \\t]+(?:@|\\w+))*)?(?=[ \\t]*r?(?:\"|'|`))",
           "beginCaptures": {
             "1": {
               "name": "renpy.meta.character.$1 variable.other.renpy",
@@ -883,16 +883,13 @@
             },
             "4": { "name": "string.quoted.renpy punctuation.definition.string.begin.renpy" }
           },
-          "end": "(?<![^\\\\]\\\\)(((?<=\\4)\\4)|\\4)([ \\t]*\\(.*?\\)(?![^\\(]*?\\)))?",
-          "endCaptures": {
-            "1": { "name": "string.quoted.renpy punctuation.definition.string.end.renpy" },
-            "2": { "name": "meta.empty-string.renpy" },
-            "3": {
-              "name": "meta.say.arguments.renpy",
-              "patterns": [ { "include": "source.renpy.python#odd-function-call" } ]
-            }
-          },
-          "patterns": [ { "include": "#strings-interior" } ]
+          "end": "$",
+          "patterns": [
+            { "include": "#strings" },
+            { "include": "source.renpy.python#odd-function-call" },
+            { "include": "#with-statement" },
+            { "include": "#fallback-patterns" }
+          ]
         },
         {
           "name": "meta.say.narrator.renpy meta.say.statement.renpy",
@@ -1012,21 +1009,34 @@
       "patterns": [
         {
           "name": "meta.image.statement.renpy",
-          "begin": "(?<=^[ \\t]*)(image)\\b[ \\t]*",
+          "begin": "(?<=^[ \\t]*)(image)\\b",
           "beginCaptures": {
             "1": { "name": "keyword.image.renpy" }
           },
-          "end": "(?=\\b(at|with)\\b|#|=|:)|$",
+          "end": "(?=:)|$",
           "patterns": [
             { "include": "#strings" },
             {
-              "match": "\\b(?:[a-zA-Z_0-9]+)\\b[ \\t]*",
+              "match": "\\b(?:[a-zA-Z_0-9]+)\\b",
               "name": "entity.name.type.image.renpy"
-            }
+            },
+            { "include": "#at-statement" },
+            { "include": "#with-statement" },
+            {
+              "contentName": "meta.python.expression.renpy",
+              "begin": "=",
+              "beginCaptures": {
+                "0": { "name": "keyword.operator.assignment.renpy" }
+              },
+              "end": "$",
+              "patterns": [
+                { "include": "source.renpy.python#expression" },
+                { "include": "#whitespace" }
+              ]
+            },
+            { "include": "#fallback-patterns" }
           ]
-        },
-        { "include": "#at-statement" },
-        { "include": "#with-statement" }
+        }
       ]
     },
 
@@ -1374,7 +1384,7 @@
     "label": {
       "name": "meta.label.statement.renpy",
       "begin": "(?<=^[ \\t]*)(label)\\b",
-      "end": "(?=#|:)|$",
+      "end": "(?=:)|$",
       "beginCaptures": {
         "1": { "name": "storage.type.function.label.renpy" }
       },
@@ -1384,7 +1394,8 @@
           "match": "\\b(?<!\\.)(hide)\\b"
         },
         { "include": "#label-def-name" },
-        { "include": "source.renpy.python#parameters" }
+        { "include": "source.renpy.python#parameters" },
+        { "include": "#fallback-patterns" }
       ]
     },
     "return-statements": {