diff options
author | David Lord <davidism@gmail.com> | 2017-07-04 09:54:18 -0700 |
---|---|---|
committer | David Lord <davidism@gmail.com> | 2017-07-04 10:00:32 -0700 |
commit | fb1e45315ddde51916cba01ed378fb9b8d12323e (patch) | |
tree | 4ca9c2a04732fafeb8ebb69104d82338cfc4dd07 | |
parent | 1f1f031c2ec7524fd7fb6b3ba264f045704f432d (diff) | |
download | jinja2-feature/kill-stringdefs.tar.gz |
go back to generating regex, simplifiedfeature/kill-stringdefs
new version uses ~2KB vs 200KB memory, is ~100x faster to load
move script to generate pattern to scripts directory
add more tests
-rw-r--r-- | jinja2/_identifier.py | 2 | ||||
-rw-r--r-- | jinja2/lexer.py | 21 | ||||
-rwxr-xr-x | scripts/generate_identifier_pattern.py | 77 | ||||
-rw-r--r-- | tests/test_lexnparse.py | 12 |
4 files changed, 100 insertions, 12 deletions
diff --git a/jinja2/_identifier.py b/jinja2/_identifier.py new file mode 100644 index 0000000..2eac35d --- /dev/null +++ b/jinja2/_identifier.py @@ -0,0 +1,2 @@ +# generated by scripts/generate_identifier_pattern.py +pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏_𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯' diff --git a/jinja2/lexer.py b/jinja2/lexer.py index 6c2b232..679e3f2 100644 --- a/jinja2/lexer.py +++ b/jinja2/lexer.py @@ -32,22 +32,25 @@ string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) integer_re = re.compile(r'\d+') -# we use the unicode identifier rule if this python version is able -# to handle unicode identifiers, otherwise the standard ASCII one. try: + # check if this Python supports Unicode identifiers compile('föö', '<unknown>', 'eval') except SyntaxError: + # no Unicode support, use ASCII identifiers name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*') check_ident = False else: - # for whatever reason these do not exist in 'some' verisons of - # python 3 within \w so we just add them. These are - # 0x1885 MONGOLIAN LETTER ALI GALI BALUDA - # 0x1886 MONGOLIAN LETTER ALI GALI THREE BALUDA - # 0x2118 SCRIPT CAPITAL P - # 0x212e ESTIMATED SYMBOL - name_re = re.compile(r'[\w\u1885\u1886\u2118\u212e]+') + # Unicode support, build a pattern to match valid characters, and set flag + # to use str.isidentifier to validate during lexing + from jinja2 import _identifier + name_re = re.compile(r'[\w{0}]+'.format(_identifier.pattern)) check_ident = True + # remove the pattern from memory after building the regex + import sys + del sys.modules['jinja2._identifier'] + import jinja2 + del jinja2._identifier + del _identifier float_re = re.compile(r'(?<!\.)\d+\.\d+') newline_re = re.compile(r'(\r\n|\r|\n)') diff --git a/scripts/generate_identifier_pattern.py b/scripts/generate_identifier_pattern.py new file mode 100755 index 0000000..7db5f4a --- /dev/null +++ b/scripts/generate_identifier_pattern.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +import itertools +import os +import re +import sys + +if sys.version_info[0] < 3: + raise RuntimeError('This needs to run on Python 3.') + + +def get_characters(): + """Find every Unicode character that is valid in a Python `identifier`_ but + is not matched by the regex ``\w`` group. + + ``\w`` matches some characters that aren't valid in identifiers, but + :meth:`str.isidentifier` will catch that later in lexing. + + All start characters are valid continue characters, so we only test for + continue characters. + + _identifier: https://docs.python.org/3/reference/lexical_analysis.html#identifiers + """ + for cp in range(sys.maxunicode + 1): + s = chr(cp) + + if ('a' + s).isidentifier() and not re.match(r'\w', s): + yield s + + +def collapse_ranges(data): + """Given a sorted list of unique characters, generate ranges representing + sequential code points. + + Source: https://stackoverflow.com/a/4629241/400617 + """ + for a, b in itertools.groupby( + enumerate(data), + lambda x: ord(x[1]) - x[0] + ): + b = list(b) + yield b[0][1], b[-1][1] + + +def build_pattern(ranges): + """Output the regex pattern for ranges of characters. + + One and two character ranges output the individual characters. + """ + out = [] + + for a, b in ranges: + if a == b: # single char + out.append(a) + elif ord(b) - ord(a) == 1: # two chars, range is redundant + out.append(a) + out.append(b) + else: + out.append(f'{a}-{b}') + + return ''.join(out) + + +def main(): + """Build the regex pattern and write it to the file + :file:`jinja2/_identifier.py`.""" + pattern = build_pattern(collapse_ranges(get_characters())) + filename = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', 'jinja2', '_identifier.py' + )) + + with open(filename, 'w', encoding='utf8') as f: + f.write('# generated by scripts/generate_identifier_pattern.py\n') + f.write(f'pattern = \'{pattern}\'\n') + + +if __name__ == '__main__': + main() diff --git a/tests/test_lexnparse.py b/tests/test_lexnparse.py index b819627..7da72c0 100644 --- a/tests/test_lexnparse.py +++ b/tests/test_lexnparse.py @@ -131,18 +131,24 @@ class TestLexer(object): (u'föö', False, True), (u'き', False, True), (u'_', True, True), - (u'1a', False, False), - # special cases in addition to \w + (u'1a', False, False), # invalid ascii start + (u'a-', False, False), # invalid ascii continue + (u'🐍', False, False), # invalid unicode start + (u'a🐍', False, False), # invalid unicode continue + # start characters not matched by \w (u'\u1885', False, True), (u'\u1886', False, True), (u'\u2118', False, True), (u'\u212e', False, True), + # continue character not matched by \w + (u'\xb7', False, False), + (u'a\xb7', False, True), )) def test_name(self, env, name, valid2, valid3): t = u'{{ ' + name + u' }}' if (valid2 and PY2) or (valid3 and not PY2): - # shouldn't raise + # valid for version being tested, shouldn't raise env.from_string(t) else: pytest.raises(TemplateSyntaxError, env.from_string, t) |