Add Unicode support.

Based on godotengine/godot#71676

Author: HolonProduction

gdtoolkit/parser/gdscript.lark

@@ -267,11 +267,13 @@ _NL: ( /\r?\n[\t ]*/ | COMMENT )+
 COMMENT: /#[^\n]*/
 LINE_CONT: /\\[\t \f]*\r?\n/
 
+// UAX#31 with GDScript modifications.
+NAME: /[\p{XID_Start}_\u037a-\u037d\u309b-\u309f\ufe70-\ufe74\ufe76-\ufefc\u0000][\p{XID_Continue}_\u037a-\u037d\ufe76-\ufefc\u0000]*/
+
 %declare _INDENT _DEDENT
 
 %ignore WS_INLINE
 %ignore COMMENT
 %ignore LINE_CONT
 
 %import common.WS_INLINE
-%import common.CNAME -> NAME

gdtoolkit/parser/parser.py

@@ -93,6 +93,7 @@ def _get_parser(
             propagate_positions=add_metadata,
             maybe_placeholders=False,
             cache=cache_filepath,
+            regex=True,
         )
 
         return a_parser

setup.py

@@ -31,7 +31,7 @@
     },
     include_package_data=True,
     install_requires=[
-        "lark-parser==0.12.0",
+        "lark-parser[regex]==0.12.0",
         "docopt>=0.6.2",
         "pyyaml>=5.1",
         "radon>=5.1",

stubs/lark/__init__.pyi

@@ -39,7 +39,7 @@ class Tree:
     def pretty(self, indent_str: str = ...) -> str: ...
 
 class Lark:
-    def __init__(self, grammar: Any): ...
+    def __init__(self, grammar: Any, **options): ...
     @classmethod
     def open(
         cls: Any, grammar_filename: str, rel_to: Optional[str] = ..., **options

tests/generated/test_expression_parsing.py

@@ -1,3 +1,5 @@
+import re
+
 import pytest
 import hypothesis.extra.lark
 from hypothesis import settings, given, HealthCheck
@@ -16,6 +18,14 @@
         '["+"] atom',
         '["+"] " " atom " "',
     )
+    # hypothesis does not support the \p syntax
+    simplified_gdscript_grammar = re.sub(
+        r"NAME:.*$",
+        r"%import common.CNAME -> NAME",
+        simplified_gdscript_grammar,
+        flags=re.MULTILINE,
+    )
+    print(simplified_gdscript_grammar)
     for keyword in ["as", "not", "await", "and", "else", "if", "or", "in"]:
         simplified_gdscript_grammar = simplified_gdscript_grammar.replace(
             f'"{keyword}"', f'" {keyword} "'
@@ -27,7 +37,7 @@
     simplified_gdscript_grammar = simplified_gdscript_grammar.replace(
         "%ignore COMMENT", ""
     )
-    gdscript_lark = Lark(simplified_gdscript_grammar)
+    gdscript_lark = Lark(simplified_gdscript_grammar, regex=True)
 
 
 def format_and_check_safety(input_code):