1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
from __future__ import unicode_literals, print_function, absolute_import, division, generators, nested_scopes
import logging
import unittest
from ply.lex import LexToken
from jsonpath_rw.lexer import JsonPathLexer, JsonPathLexerError
class TestLexer(unittest.TestCase):
def token(self, value, ty=None):
t = LexToken()
t.type = ty if ty != None else value
t.value = value
t.lineno = -1
t.lexpos = -1
return t
def assert_lex_equiv(self, s, stream2):
# NOTE: lexer fails to reset after call?
l = JsonPathLexer(debug=True)
stream1 = list(l.tokenize(s)) # Save the stream for debug output when a test fails
stream2 = list(stream2)
assert len(stream1) == len(stream2)
for token1, token2 in zip(stream1, stream2):
print(token1, token2)
assert token1.type == token2.type
assert token1.value == token2.value
@classmethod
def setup_class(cls):
logging.basicConfig()
def test_simple_inputs(self):
self.assert_lex_equiv('$', [self.token('$', '$')])
self.assert_lex_equiv('"hello"', [self.token('hello', 'ID')])
self.assert_lex_equiv("'goodbye'", [self.token('goodbye', 'ID')])
self.assert_lex_equiv("'doublequote\"'", [self.token('doublequote"', 'ID')])
self.assert_lex_equiv(r'"doublequote\""', [self.token('doublequote"', 'ID')])
self.assert_lex_equiv(r"'singlequote\''", [self.token("singlequote'", 'ID')])
self.assert_lex_equiv('"singlequote\'"', [self.token("singlequote'", 'ID')])
self.assert_lex_equiv('fuzz', [self.token('fuzz', 'ID')])
self.assert_lex_equiv('1', [self.token(1, 'NUMBER')])
self.assert_lex_equiv('45', [self.token(45, 'NUMBER')])
self.assert_lex_equiv('-1', [self.token(-1, 'NUMBER')])
self.assert_lex_equiv(' -13 ', [self.token(-13, 'NUMBER')])
self.assert_lex_equiv('"fuzz.bang"', [self.token('fuzz.bang', 'ID')])
self.assert_lex_equiv('fuzz.bang', [self.token('fuzz', 'ID'), self.token('.', '.'), self.token('bang', 'ID')])
self.assert_lex_equiv('fuzz.*', [self.token('fuzz', 'ID'), self.token('.', '.'), self.token('*', '*')])
self.assert_lex_equiv('fuzz..bang', [self.token('fuzz', 'ID'), self.token('..', 'DOUBLEDOT'), self.token('bang', 'ID')])
self.assert_lex_equiv('&', [self.token('&', '&')])
self.assert_lex_equiv('@', [self.token('@', 'ID')])
self.assert_lex_equiv('`this`', [self.token('this', 'NAMED_OPERATOR')])
self.assert_lex_equiv('|', [self.token('|', '|')])
self.assert_lex_equiv('where', [self.token('where', 'WHERE')])
def test_basic_errors(self):
def tokenize(s):
l = JsonPathLexer(debug=True)
return list(l.tokenize(s))
self.assertRaises(JsonPathLexerError, tokenize, "'\"")
self.assertRaises(JsonPathLexerError, tokenize, '"\'')
self.assertRaises(JsonPathLexerError, tokenize, '`"')
self.assertRaises(JsonPathLexerError, tokenize, "`'")
self.assertRaises(JsonPathLexerError, tokenize, '"`')
self.assertRaises(JsonPathLexerError, tokenize, "'`")
self.assertRaises(JsonPathLexerError, tokenize, '?')
self.assertRaises(JsonPathLexerError, tokenize, '$.foo.bar.#')
|
