Fix priority of JSONPath lexer rules. (#39)

Author: jg-rp

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Python JSONPath Change Log
 
+## Version 0.10.1
+
+**Hot fix**
+
+- Fixed priority of JSONPath lexer rules. Previously, standard short tokens (like `*` and `?`) had a higher priority than environment-controlled tokens (like `JSONPathEnvironment.keys_selector_token`), making it impossible to incorporate short token characters into longer environment-controlled tokens.
+
 ## Version 0.10.0
 
 **Breaking Changes**

jsonpath/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2023-present James Prior <jamesgr.prior@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.10.0"
+__version__ = "0.10.1"

jsonpath/lex.py

@@ -108,9 +108,7 @@ def compile_rules(self) -> Pattern[str]:
             (TOKEN_DOUBLE_QUOTE_STRING, self.double_quote_pattern),
             (TOKEN_SINGLE_QUOTE_STRING, self.single_quote_pattern),
             (TOKEN_RE_PATTERN, self.re_pattern),
-            (TOKEN_WILD, r"\*"),
             (TOKEN_LIST_SLICE, self.slice_list_pattern),
-            (TOKEN_FILTER, r"\?"),
             (TOKEN_FUNCTION, self.function_pattern),
             (TOKEN_DOT_PROPERTY, self.dot_property_pattern),
             (TOKEN_FLOAT, r"-?\d+\.\d*(?:e[+-]?\d+)?"),
@@ -125,6 +123,8 @@ def compile_rules(self) -> Pattern[str]:
             (TOKEN_INTERSECTION, re.escape(self.env.intersection_token)),
             (TOKEN_FILTER_CONTEXT, re.escape(self.env.filter_context_token)),
             (TOKEN_KEYS, re.escape(self.env.keys_selector_token)),
+            (TOKEN_WILD, r"\*"),
+            (TOKEN_FILTER, r"\?"),
             (TOKEN_IN, r"in"),
             (TOKEN_TRUE, r"[Tt]rue"),
             (TOKEN_FALSE, r"[Ff]alse"),

tests/test_env.py

@@ -158,3 +158,15 @@ def test_no_unicode_escape() -> None:
     env = JSONPathEnvironment(unicode_escape=False)
     assert env.findall(selector, document) == []
     assert env.findall(selector, {"\\uD834\\uDD1E": "B"}) == ["B"]
+
+
+def test_custom_keys_selector_token() -> None:
+    """Test that we can change the non-standard keys selector."""
+
+    class MyJSONPathEnvironment(JSONPathEnvironment):
+        keys_selector_token = "*~"
+
+    env = MyJSONPathEnvironment()
+    data = {"foo": {"a": 1, "b": 2, "c": 3}}
+    assert env.findall("$.foo.*~", data) == ["a", "b", "c"]
+    assert env.findall("$.foo.*", data) == [1, 2, 3]