Refactor tests/test_lexer.py

Author: kurtmckee

tests/test_lexer.py

@@ -1,68 +1,55 @@
-import logging
-import unittest
-
-from ply.lex import LexToken
+import pytest
 
 from jsonpath_ng.lexer import JsonPathLexer, JsonPathLexerError
 
-class TestLexer(unittest.TestCase):
-
-    def token(self, value, ty=None):
-        t = LexToken()
-        t.type = ty if ty != None else value
-        t.value = value
-        t.lineno = -1
-        t.lexpos = -1
-        return t
-
-    def assert_lex_equiv(self, s, stream2):
-        # NOTE: lexer fails to reset after call?
-        l = JsonPathLexer(debug=True)
-        stream1 = list(l.tokenize(s)) # Save the stream for debug output when a test fails
-        stream2 = list(stream2)
-        assert len(stream1) == len(stream2)
-        for token1, token2 in zip(stream1, stream2):
-            print(token1, token2)
-            assert token1.type  == token2.type
-            assert token1.value == token2.value
-
-    @classmethod
-    def setup_class(cls):
-        logging.basicConfig()
-
-    def test_simple_inputs(self):
-        self.assert_lex_equiv('$', [self.token('$', '$')])
-        self.assert_lex_equiv('"hello"', [self.token('hello', 'ID')])
-        self.assert_lex_equiv("'goodbye'", [self.token('goodbye', 'ID')])
-        self.assert_lex_equiv("'doublequote\"'", [self.token('doublequote"', 'ID')])
-        self.assert_lex_equiv(r'"doublequote\""', [self.token('doublequote"', 'ID')])
-        self.assert_lex_equiv(r"'singlequote\''", [self.token("singlequote'", 'ID')])
-        self.assert_lex_equiv('"singlequote\'"', [self.token("singlequote'", 'ID')])
-        self.assert_lex_equiv('fuzz', [self.token('fuzz', 'ID')])
-        self.assert_lex_equiv('1', [self.token(1, 'NUMBER')])
-        self.assert_lex_equiv('45', [self.token(45, 'NUMBER')])
-        self.assert_lex_equiv('-1', [self.token(-1, 'NUMBER')])
-        self.assert_lex_equiv(' -13 ', [self.token(-13, 'NUMBER')])
-        self.assert_lex_equiv('"fuzz.bang"', [self.token('fuzz.bang', 'ID')])
-        self.assert_lex_equiv('fuzz.bang', [self.token('fuzz', 'ID'), self.token('.', '.'), self.token('bang', 'ID')])
-        self.assert_lex_equiv('fuzz.*', [self.token('fuzz', 'ID'), self.token('.', '.'), self.token('*', '*')])
-        self.assert_lex_equiv('fuzz..bang', [self.token('fuzz', 'ID'), self.token('..', 'DOUBLEDOT'), self.token('bang', 'ID')])
-        self.assert_lex_equiv('&', [self.token('&', '&')])
-        self.assert_lex_equiv('@', [self.token('@', 'ID')])
-        self.assert_lex_equiv('`this`', [self.token('this', 'NAMED_OPERATOR')])
-        self.assert_lex_equiv('|', [self.token('|', '|')])
-        self.assert_lex_equiv('where', [self.token('where', 'WHERE')])
-
-    def test_basic_errors(self):
-        def tokenize(s):
-            l = JsonPathLexer(debug=True)
-            return list(l.tokenize(s))
-
-        self.assertRaises(JsonPathLexerError, tokenize, "'\"")
-        self.assertRaises(JsonPathLexerError, tokenize, '"\'')
-        self.assertRaises(JsonPathLexerError, tokenize, '`"')
-        self.assertRaises(JsonPathLexerError, tokenize, "`'")
-        self.assertRaises(JsonPathLexerError, tokenize, '"`')
-        self.assertRaises(JsonPathLexerError, tokenize, "'`")
-        self.assertRaises(JsonPathLexerError, tokenize, '?')
-        self.assertRaises(JsonPathLexerError, tokenize, '$.foo.bar.#')
+token_test_cases = (
+    ("$", (("$", "$"),)),
+    ('"hello"', (("hello", "ID"),)),
+    ("'goodbye'", (("goodbye", "ID"),)),
+    ("'doublequote\"'", (('doublequote"', "ID"),)),
+    (r'"doublequote\""', (('doublequote"', "ID"),)),
+    (r"'singlequote\''", (("singlequote'", "ID"),)),
+    ('"singlequote\'"', (("singlequote'", "ID"),)),
+    ("fuzz", (("fuzz", "ID"),)),
+    ("1", ((1, "NUMBER"),)),
+    ("45", ((45, "NUMBER"),)),
+    ("-1", ((-1, "NUMBER"),)),
+    (" -13 ", ((-13, "NUMBER"),)),
+    ('"fuzz.bang"', (("fuzz.bang", "ID"),)),
+    ("fuzz.bang", (("fuzz", "ID"), (".", "."), ("bang", "ID"))),
+    ("fuzz.*", (("fuzz", "ID"), (".", "."), ("*", "*"))),
+    ("fuzz..bang", (("fuzz", "ID"), ("..", "DOUBLEDOT"), ("bang", "ID"))),
+    ("&", (("&", "&"),)),
+    ("@", (("@", "ID"),)),
+    ("`this`", (("this", "NAMED_OPERATOR"),)),
+    ("|", (("|", "|"),)),
+    ("where", (("where", "WHERE"),)),
+)
+
+
+@pytest.mark.parametrize("string, expected_token_info", token_test_cases)
+def test_lexer(string, expected_token_info):
+    lexer = JsonPathLexer(debug=True)
+    tokens = list(lexer.tokenize(string))
+    assert len(tokens) == len(expected_token_info)
+    for token, (expected_value, expected_type) in zip(tokens, expected_token_info):
+        assert token.type == expected_type
+        assert token.value == expected_value
+
+
+invalid_token_test_cases = (
+    "'\"",
+    "\"'",
+    '`"',
+    "`'",
+    '"`',
+    "'`",
+    "?",
+    "$.foo.bar.#",
+)
+
+
+@pytest.mark.parametrize("string", invalid_token_test_cases)
+def test_lexer_errors(string):
+    with pytest.raises(JsonPathLexerError):
+        list(JsonPathLexer().tokenize(string))