Lkt: fix the handling of lexing rules with many matchers

Author: pmderodat

langkit/dsl_unparse.py

@@ -1872,7 +1872,7 @@ def unparse_rule_matchers(rule_set):
             return unparse_matcher(rule_set[0][0].matcher)
         elif len(rule_set) > 1:
             return "or($i$hl{}$d$hl)".format("$hl".join(
-                unparse_matcher(rule.matcher) for rule, _ in rule_set
+                f"| {unparse_matcher(rule.matcher)}" for rule, _ in rule_set
             ))
         return "bordel"
 

langkit/lkt_lowering.py

@@ -1511,11 +1511,12 @@ def process_token_rule(
             assert isinstance(r.f_decl, L.GrammarRuleDecl)
             matcher_expr = r.f_decl.f_expr
             if matcher_expr is not None:
-                rule = (lower_matcher(matcher_expr), token)
-                if is_pre:
-                    pre_rules.append(rule)
-                else:
-                    rules.append(rule)
+                for matcher in lower_matcher_list(matcher_expr):
+                    rule = (matcher, token)
+                    if is_pre:
+                        pre_rules.append(rule)
+                    else:
+                        rules.append(rule)
 
     def process_pattern(full_decl: L.FullDecl) -> None:
         """
@@ -1545,6 +1546,25 @@ def process_pattern(full_decl: L.FullDecl) -> None:
                 decl.f_val.p_denoted_value, Location.from_lkt_node(decl)
             )
 
+    def lower_matcher_list(expr: L.GrammarExpr) -> list[Matcher]:
+        """
+        Lower a list of token matchers to our internals.
+
+        Lists of token matchers are made up of "atomic" matchers nested in "or"
+        grammar expressions.
+        """
+        if isinstance(expr, L.GrammarOrExpr):
+            result = []
+            for child_list in expr.f_sub_exprs:
+                with ctx.lkt_context(child_list):
+                    check_source_language(
+                        len(child_list) == 1, "exactly one matcher expected"
+                    )
+                result += lower_matcher_list(child_list[0])
+            return result
+        else:
+            return [lower_matcher(expr)]
+
     def lower_matcher(expr: L.GrammarExpr) -> Matcher:
         """
         Lower a token matcher to our internals.

testsuite/tests/grammar/invalid_lexers/or_1.lkt

@@ -0,0 +1,10 @@
+import common
+
+lexer foo_lexer {
+    example <- or("example" "foo" | "bar")
+}
+
+@with_lexer(foo_lexer)
+grammar foo_grammar {
+    @main_rule main_rule <- Example("example")
+}

testsuite/tests/grammar/invalid_lexers/or_2.lkt

@@ -0,0 +1,10 @@
+import common
+
+lexer foo_lexer {
+    example <- or("example" | or ("foo" | /))
+}
+
+@with_lexer(foo_lexer)
+grammar foo_grammar {
+    @main_rule main_rule <- Example("example")
+}

testsuite/tests/grammar/invalid_lexers/test.out

@@ -55,6 +55,18 @@ family_1.lkt:4:5: error: Only lexer rules allowed in family blocks
 == foo.lkt ==
 Code generation was successful
 
+== or_1.lkt ==
+or_1.lkt:4:19: error: exactly one matcher expected
+4 |     example <- or("example" "foo" | "bar")
+  |                   ^^^^^^^^^^^^^^^
+
+
+== or_2.lkt ==
+or_2.lkt:4:43: error: Invalid lexing expression
+4 |     example <- or("example" | or ("foo" | /))
+  |                                           ^
+
+
 == pattern_1.lkt ==
 pattern_1.lkt:4:16: error: unbalanced square bracket
 4 |     example <- p"["

testsuite/tests/grammar/token_or/expected_concrete_syntax.lkt

@@ -0,0 +1,28 @@
+lexer foo_lexer {
+
+    @trivia() whitespace <- p"[ \n\r\t]+"
+    id <- p"[a-zA-Z]+"
+    number <- p"[0-9]+"
+    assign <- or(
+        | "="
+        | ":="
+    )
+}
+@with_lexer(foo_lexer)
+grammar foo_grammar {
+    @main_rule expr <- list*(Assignment(Identifier(@id) "=" Number(@number)))
+}
+
+@abstract class FooNode implements Node[FooNode] {
+}
+
+class Assignment: FooNode {
+    @parse_field name: Identifier
+    @parse_field value: Number
+}
+
+class Identifier: FooNode implements TokenNode {
+}
+
+class Number: FooNode implements TokenNode {
+}

testsuite/tests/grammar/token_or/main.py

@@ -0,0 +1,21 @@
+import sys
+
+import libfoolang
+
+
+print("main.py: Running...")
+
+text = b"""\
+a = 1
+b := 24
+"""
+
+ctx = libfoolang.AnalysisContext()
+u = ctx.get_from_buffer("main.txt", buffer=text)
+if u.diagnostics:
+    for d in u.diagnostics:
+        print(d)
+    sys.exit(1)
+u.root.dump()
+
+print("main.py: Done.")

testsuite/tests/grammar/token_or/test.out

@@ -0,0 +1,16 @@
+main.py: Running...
+AssignmentList main.txt:1:1-2:8
+|item_0:
+|  Assignment main.txt:1:1-1:6
+|  |f_name:
+|  |  Identifier main.txt:1:1-1:2: a
+|  |f_value:
+|  |  Number main.txt:1:5-1:6: 1
+|item_1:
+|  Assignment main.txt:2:1-2:8
+|  |f_name:
+|  |  Identifier main.txt:2:1-2:2: b
+|  |f_value:
+|  |  Number main.txt:2:6-2:8: 24
+main.py: Done.
+Done

testsuite/tests/grammar/token_or/test.py

@@ -0,0 +1,34 @@
+"""
+Check that the case insenvitity feature works as expected.
+"""
+
+from langkit.compiled_types import T
+from langkit.dsl import ASTNode, Field
+
+from utils import build_and_run, unparse_all_script
+
+
+class FooNode(ASTNode):
+    pass
+
+
+class Assignment(FooNode):
+    name = Field(type=T.Identifier)
+    value = Field(type=T.Number)
+
+
+class Number(FooNode):
+    token_node = True
+
+
+class Identifier(FooNode):
+    token_node = True
+
+
+build_and_run(
+    lkt_file='expected_concrete_syntax.lkt',
+    unparse_script=unparse_all_script,
+    py_script='main.py',
+    types_from_lkt=True,
+)
+print('Done')

testsuite/tests/grammar/token_or/test.yaml

@@ -0,0 +1 @@
+driver: python